From 2d7a84684312fc1c46043fa10af0ea336e73f11d Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 18 Mar 2024 15:16:17 +0800 Subject: HTTP: Added variable validation to the response_headers option This is to improve error messages for response headers configuration. Take the configuration as an example: { "response_headers": { "a": "$b" } } Previously, when applying it the user would see this error message: failed to apply previous configuration After this change, the user will see this improved error message: the previous configuration is invalid: Unknown variable "b" in the "a" value --- src/nxt_conf_validation.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 2099f887..8f802fb7 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -2572,6 +2572,7 @@ static nxt_int_t nxt_conf_vldt_response_header(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value) { + nxt_str_t str; nxt_uint_t type; static nxt_str_t content_length = nxt_string("Content-Length"); @@ -2588,7 +2589,17 @@ nxt_conf_vldt_response_header(nxt_conf_validation_t *vldt, nxt_str_t *name, type = nxt_conf_type(value); - if (type == NXT_CONF_STRING || type == NXT_CONF_NULL) { + if (type == NXT_CONF_NULL) { + return NXT_OK; + } + + if (type == NXT_CONF_STRING) { + nxt_conf_get_string(value, &str); + + if (nxt_is_tstr(&str)) { + return nxt_conf_vldt_var(vldt, name, &str); + } + return NXT_OK; } -- cgit From e5bc299d7a55a66e1ecf54d35dcdd9448c49f3d4 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 16 Apr 2024 19:22:59 +0100 Subject: configuration: Constify numerous pointers Mark numerous function argument pointers as 'const' in the configuration sub-system. This also does the same with a few functions in src/nxt_conf_validation.c that are required to accomplish the below, attacking the rest is an exercise for another day... While this is a worthwhile hardening exercise in its own right, the main impetus for this is to 'constify' some local function variables which are currently defined with 'static' storage class and turn them into 'static const', which will be done in a subsequent patch. Reviewed-by: Zhidao HONG Signed-off-by: Andrew Clayton --- src/nxt_conf_validation.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8f802fb7..38a918fb 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -73,11 +73,11 @@ struct nxt_conf_vldt_object_s { static nxt_int_t nxt_conf_vldt_type(nxt_conf_validation_t *vldt, - nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); + const nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); static nxt_int_t nxt_conf_vldt_error(nxt_conf_validation_t *vldt, const char *fmt, ...); -static nxt_int_t nxt_conf_vldt_var(nxt_conf_validation_t *vldt, nxt_str_t *name, - nxt_str_t *value); +static nxt_int_t nxt_conf_vldt_var(nxt_conf_validation_t *vldt, + const nxt_str_t *name, nxt_str_t *value); static nxt_int_t nxt_conf_vldt_if(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); nxt_inline nxt_int_t nxt_conf_vldt_unsupported(nxt_conf_validation_t *vldt, @@ -1436,7 +1436,7 @@ nxt_conf_validate(nxt_conf_validation_t *vldt) static nxt_int_t -nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, +nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type) { u_char *p; @@ -1548,7 +1548,7 @@ nxt_conf_vldt_unsupported(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, static nxt_int_t -nxt_conf_vldt_var(nxt_conf_validation_t *vldt, nxt_str_t *name, +nxt_conf_vldt_var(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_str_t *value) { u_char error[NXT_MAX_ERROR_STR]; -- cgit From 8f861cf4d15e8befca6edcee4b04b5304f082f05 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 16 Apr 2024 20:30:48 +0100 Subject: Constify a bunch of static local variables A common pattern was to declare variables in functions like static nxt_str_t ... Not sure why static, as they were being treated more like string literals (and of course they are _not_ thread safe), let's actually make them constants (qualifier wise). This handles core code conversion. Reviewed-by: Zhidao HONG Signed-off-by: Andrew Clayton --- src/nxt_conf_validation.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 38a918fb..4aaa1b9a 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -1445,7 +1445,7 @@ nxt_conf_vldt_type(nxt_conf_validation_t *vldt, const nxt_str_t *name, nxt_uint_t value_type, n, t; u_char buf[nxt_length(NXT_CONF_VLDT_ANY_TYPE_STR)]; - static nxt_str_t type_name[] = { + static const nxt_str_t type_name[] = { nxt_string("a null"), nxt_string("a boolean"), nxt_string("an integer number"), @@ -1568,7 +1568,7 @@ nxt_conf_vldt_if(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, { nxt_str_t str; - static nxt_str_t if_str = nxt_string("if"); + static const nxt_str_t if_str = nxt_string("if"); if (nxt_conf_type(value) != NXT_CONF_STRING) { return nxt_conf_vldt_error(vldt, "The \"if\" must be a string"); @@ -1731,7 +1731,7 @@ nxt_conf_vldt_action(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_conf_value_t *action; nxt_conf_vldt_object_t *members; - static struct { + static const struct { nxt_str_t name; nxt_conf_vldt_object_t *members; @@ -1778,7 +1778,7 @@ nxt_conf_vldt_pass(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_int_t ret; nxt_str_t segments[3]; - static nxt_str_t targets_str = nxt_string("targets"); + static const nxt_str_t targets_str = nxt_string("targets"); nxt_conf_get_string(value, &pass); @@ -1932,7 +1932,7 @@ nxt_conf_vldt_share_element(nxt_conf_validation_t *vldt, { nxt_str_t str; - static nxt_str_t share = nxt_string("share"); + static const nxt_str_t share = nxt_string("share"); if (nxt_conf_type(value) != NXT_CONF_STRING) { return nxt_conf_vldt_error(vldt, "The \"share\" array must " @@ -1982,7 +1982,7 @@ nxt_conf_vldt_python(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, { nxt_conf_value_t *targets; - static nxt_str_t targets_str = nxt_string("targets"); + static const nxt_str_t targets_str = nxt_string("targets"); targets = nxt_conf_get_object_member(value, &targets_str, NULL); @@ -2575,7 +2575,7 @@ nxt_conf_vldt_response_header(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_str_t str; nxt_uint_t type; - static nxt_str_t content_length = nxt_string("Content-Length"); + static const nxt_str_t content_length = nxt_string("Content-Length"); if (name->length == 0) { return nxt_conf_vldt_error(vldt, "The response header name " @@ -2615,7 +2615,7 @@ nxt_conf_vldt_app_name(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_str_t name; nxt_conf_value_t *apps, *app; - static nxt_str_t apps_str = nxt_string("applications"); + static const nxt_str_t apps_str = nxt_string("applications"); nxt_conf_get_string(value, &name); @@ -2647,8 +2647,8 @@ nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, { nxt_conf_value_t *client_ip, *protocol; - static nxt_str_t client_ip_str = nxt_string("client_ip"); - static nxt_str_t protocol_str = nxt_string("protocol"); + static const nxt_str_t client_ip_str = nxt_string("client_ip"); + static const nxt_str_t protocol_str = nxt_string("protocol"); client_ip = nxt_conf_get_object_member(value, &client_ip_str, NULL); protocol = nxt_conf_get_object_member(value, &protocol_str, NULL); @@ -2673,9 +2673,9 @@ nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *type_value; nxt_app_lang_module_t *lang; - static nxt_str_t type_str = nxt_string("type"); + static const nxt_str_t type_str = nxt_string("type"); - static struct { + static const struct { nxt_conf_vldt_handler_t validator; nxt_conf_vldt_object_t *members; @@ -3188,7 +3188,7 @@ nxt_conf_vldt_php(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, { nxt_conf_value_t *targets; - static nxt_str_t targets_str = nxt_string("targets"); + static const nxt_str_t targets_str = nxt_string("targets"); targets = nxt_conf_get_object_member(value, &targets_str, NULL); @@ -3269,7 +3269,7 @@ nxt_conf_vldt_upstream(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_int_t ret; nxt_conf_value_t *conf; - static nxt_str_t servers = nxt_string("servers"); + static const nxt_str_t servers = nxt_string("servers"); ret = nxt_conf_vldt_type(vldt, name, value, NXT_CONF_VLDT_OBJECT); @@ -3414,7 +3414,7 @@ nxt_conf_vldt_access_log(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, nxt_int_t ret; nxt_conf_vldt_access_log_conf_t conf; - static nxt_str_t format_str = nxt_string("format"); + static const nxt_str_t format_str = nxt_string("format"); if (nxt_conf_type(value) == NXT_CONF_STRING) { return NXT_OK; -- cgit From 64f4c78bf441fa9e021d905a03d374d0a9e05e8d Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Tue, 11 Jun 2024 17:59:00 +0800 Subject: http: Support chunked request bodies This is a temporary support for chunked request bodies by converting to Content-Length. This allows for processing of such requests until a more permanent solution is developed. A new configuration option "chunked_transform" has been added to enable this feature. The option can be set as follows: { "settings": { "chunked_transform": true } } By default, this option is set to false, which retains the current behaviour of rejecting chunked requests with a '411 Length Required' status code. Please note that this is an experimental implementation. Reviewed-by: Andrew Clayton --- src/nxt_conf_validation.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 4aaa1b9a..f91fc887 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -368,6 +368,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("server_version"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("chunked_transform"), + .type = NXT_CONF_VLDT_BOOLEAN, }, NXT_CONF_VLDT_END -- cgit From a9aa9e76db2766a681350c09947df848898531f6 Mon Sep 17 00:00:00 2001 From: Gourav Date: Wed, 26 Jun 2024 11:14:50 +0530 Subject: python: Support application factories Adds support for the app factory pattern to the Python language module. A factory is a callable that returns a WSGI or ASGI application object. Unit does not support passing arguments to factories. Setting the `factory` option to `true` instructs Unit to treat the configured `callable` as a factory. For example: "my-app": { "type": "python", "path": "/srv/www/", "module": "hello", "callable": "create_app", "factory": true } This is similar to other WSGI / ASGI servers. E.g., $ uvicorn --factory hello:create_app $ gunicorn 'hello:create_app()' The factory setting defaults to false. Closes: https://github.com/nginx/unit/issues/1106 Link: [ Commit message - Dan / Minor code tweaks - Andrew ] Signed-off-by: Andrew Clayton --- src/nxt_conf_validation.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index f91fc887..04091745 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -841,6 +841,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { .type = NXT_CONF_VLDT_STRING, .validator = nxt_conf_vldt_targets_exclusive, .u.string = "callable", + }, { + .name = nxt_string("factory"), + .type = NXT_CONF_VLDT_BOOLEAN, + .validator = nxt_conf_vldt_targets_exclusive, + .u.string = "factory", }, { .name = nxt_string("prefix"), .type = NXT_CONF_VLDT_STRING, @@ -865,6 +870,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_target_members[] = { }, { .name = nxt_string("callable"), .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("factory"), + .type = NXT_CONF_VLDT_BOOLEAN, }, { .name = nxt_string("prefix"), .type = NXT_CONF_VLDT_STRING, @@ -883,6 +891,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_notargets_members[] = { }, { .name = nxt_string("callable"), .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("factory"), + .type = NXT_CONF_VLDT_BOOLEAN, }, { .name = nxt_string("prefix"), .type = NXT_CONF_VLDT_STRING, -- cgit From 57c88fd4086d47bf866e3fe7e4c03d0262d413ae Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Mon, 12 Aug 2024 22:56:16 +0100 Subject: router: Make the number of router threads configurable Unit generally creates an extra number of router threads (to handle client connections, not incl the main thread) to match the number of available CPUs. There are cases when this can go wrong, e.g on a high CPU count machine and Unit is being effectively limited to a few CPUs via the cgroups cpu controller. So Unit may create a large number of router threads when they are only going to effectively run on a couple of CPUs or so. There may be other cases where you would like to tweak the number of router threads, depending on your workload. As it turns out it looks like it was intended to be made configurable but was just never hooked up to the config system. This adds a new '/settings/listen_threads' config option which can be set like { "listen": { ... }, "settings": { "listen_threads": 2, ... }, ... } Before this patch (on a four cpu system) $ ps -efL | grep router andrew 419832 419829 419832 0 5 Aug12 pts/10 00:00:00 unit: router andrew 419832 419829 419833 0 5 Aug12 pts/10 00:00:00 unit: router andrew 419832 419829 419834 0 5 Aug12 pts/10 00:00:00 unit: router andrew 419832 419829 445145 0 5 03:31 pts/10 00:00:00 unit: router andrew 419832 419829 445146 0 5 03:31 pts/10 00:00:00 unit: router After, with a threads setting of 2 $ ps -efL | grep router andrew 419832 419829 419832 0 3 Aug12 pts/10 00:00:00 unit: router andrew 419832 419829 419833 0 3 Aug12 pts/10 00:00:00 unit: router andrew 419832 419829 419834 0 3 Aug12 pts/10 00:00:00 unit: router Closes: https://github.com/nginx/unit/issues/1042 Signed-off-by: Andrew Clayton --- src/nxt_conf_validation.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 04091745..c9c51ac1 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -134,6 +134,8 @@ static nxt_int_t nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_python_prefix(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, @@ -305,6 +307,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { { + .name = nxt_string("listen_threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_listen_threads, + }, { .name = nxt_string("http"), .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_object, @@ -2078,6 +2084,27 @@ nxt_conf_vldt_python_prefix(nxt_conf_validation_t *vldt, return NXT_OK; } +static nxt_int_t +nxt_conf_vldt_listen_threads(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t threads; + + threads = nxt_conf_get_number(value); + + if (threads < 1) { + return nxt_conf_vldt_error(vldt, "The \"listen_threads\" number must " + "be equal to or greater than 1."); + } + + if (threads > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"listen_threads\" number must " + "not exceed %d.", NXT_INT32_T_MAX); + } + + return NXT_OK; +} + static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, -- cgit From 76489fb7e0ab9142651b91ee8072c6cda270dd09 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Wed, 14 Aug 2024 00:33:13 +0100 Subject: conf, router: Make the listen(2) backlog configurable @oopsoop2 on GitHub reported a performance issue related to the default listen(2) backlog size of 511 on nginx. They found that increasing it helped, nginx has a config option to configure this. They would like to be able to do the same on Unit (which also defaults to 511 on some systems). This seems reasonable. NOTE: On Linux before commit 97c15fa38 ("socket: Use a default listen backlog of -1 on Linux") we defaulted to 511. Since that commit we default to the Kernels default, which before 5.4 is 128 and after is 4096. This adds a new per-listener 'backlog' config option, e.g { "listeners": { "[::1]:8080": { "pass": "routes", "backlog": 1024 }, } ... } This doesn't effect the control socket. Closes: https://github.com/nginx/unit/issues/1384 Reported-by: Signed-off-by: Andrew Clayton --- src/nxt_conf_validation.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index c9c51ac1..8f31bd18 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -178,6 +178,8 @@ static nxt_int_t nxt_conf_vldt_app_name(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_object(nxt_conf_validation_t *vldt, @@ -430,6 +432,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { .type = NXT_CONF_VLDT_OBJECT, .validator = nxt_conf_vldt_object, .u.members = nxt_conf_vldt_client_ip_members + }, { + .name = nxt_string("backlog"), + .type = NXT_CONF_VLDT_NUMBER, + .validator = nxt_conf_vldt_listen_backlog, }, #if (NXT_TLS) @@ -2704,6 +2710,32 @@ nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +static nxt_int_t +nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t backlog; + + backlog = nxt_conf_get_number(value); + + /* + * POSIX allows this to be 0 and some systems use -1 to + * indicate to use the OS's default value. + */ + if (backlog < -1) { + return nxt_conf_vldt_error(vldt, "The \"backlog\" number must be " + "equal to or greater than -1."); + } + + if (backlog > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"backlog\" number must " + "not exceed %d.", NXT_INT32_T_MAX); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value) -- cgit From debd61c3a4f7e5817bf842c2166217929ef80c88 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Fri, 16 Aug 2024 00:29:16 +0800 Subject: http: Add "if" option to the "match" object This feature allows users to specify conditions to check if one route is matched. It is used the same way as the "if" option in the access log. Example: { "match": { "if": "`${headers['User-Agent'].split('/')[0] == 'curl'}`" }, "action": { "return": 204 } } --- src/nxt_conf_validation.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/nxt_conf_validation.c') diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 8f31bd18..5d7f7c52 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -696,6 +696,10 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, .validator = nxt_conf_vldt_match_patterns_sets, .u.string = "cookies" + }, { + .name = nxt_string("if"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_if, }, NXT_CONF_VLDT_END -- cgit