From 35178dbb06432747f410d3b9f259d019afe85379 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 24 Nov 2020 16:40:28 +0300 Subject: Version bump. --- docs/changes.xml | 7 +++++++ version | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index 772d65f2..39c270ad 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,13 @@ + + + + + diff --git a/version b/version index 2ad4b832..55e5da42 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ # Copyright (C) NGINX, Inc. -NXT_VERSION=1.21.0 -NXT_VERNUM=12100 +NXT_VERSION=1.22.0 +NXT_VERNUM=12200 -- cgit From 956fce6614dd0979a2995c792131d894e132e614 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 24 Nov 2020 16:40:35 +0300 Subject: Libunit: improved error logging around initialization env variable. --- src/nxt_unit.c | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 097f50d6..b6904ce9 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -784,8 +784,8 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, { int rc; int ready_fd, router_fd, read_in_fd, read_out_fd; - char *unit_init, *version_end; - long version_length; + char *unit_init, *version_end, *vars; + size_t version_length; int64_t ready_pid, router_pid, read_pid; uint32_t ready_stream, router_id, ready_id, read_id; @@ -797,21 +797,30 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, return NXT_UNIT_ERROR; } - nxt_unit_debug(NULL, "%s='%s'", NXT_UNIT_INIT_ENV, unit_init); + version_end = strchr(unit_init, ';'); + if (nxt_slow_path(version_end == NULL)) { + nxt_unit_alert(NULL, "Unit version not found in %s=\"%s\"", + NXT_UNIT_INIT_ENV, unit_init); + + return NXT_UNIT_ERROR; + } - version_length = nxt_length(NXT_VERSION); + version_length = version_end - unit_init; - version_end = strchr(unit_init, ';'); - if (version_end == NULL - || version_end - unit_init != version_length - || memcmp(unit_init, NXT_VERSION, version_length) != 0) - { - nxt_unit_alert(NULL, "version check error"); + rc = version_length != nxt_length(NXT_VERSION) + || memcmp(unit_init, NXT_VERSION, nxt_length(NXT_VERSION)); + + if (nxt_slow_path(rc != 0)) { + nxt_unit_alert(NULL, "versions mismatch: the Unit daemon has version " + "%.*s, while the app was compiled with libunit %s", + (int) version_length, unit_init, NXT_VERSION); return NXT_UNIT_ERROR; } - rc = sscanf(version_end + 1, + vars = version_end + 1; + + rc = sscanf(vars, "%"PRIu32";" "%"PRId64",%"PRIu32",%d;" "%"PRId64",%"PRIu32",%d;" @@ -823,12 +832,22 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, &read_pid, &read_id, &read_in_fd, &read_out_fd, log_fd, shm_limit); + if (nxt_slow_path(rc == EOF)) { + nxt_unit_alert(NULL, "sscanf(%s) failed: %s (%d) for %s env", + vars, strerror(errno), errno, NXT_UNIT_INIT_ENV); + + return NXT_UNIT_ERROR; + } + if (nxt_slow_path(rc != 13)) { - nxt_unit_alert(NULL, "failed to scan variables: %d", rc); + nxt_unit_alert(NULL, "invalid number of variables in %s env: " + "found %d of %d in %s", NXT_UNIT_INIT_ENV, rc, 13, vars); return NXT_UNIT_ERROR; } + nxt_unit_debug(NULL, "%s='%s'", NXT_UNIT_INIT_ENV, unit_init); + nxt_unit_port_id_init(&ready_port->id, (pid_t) ready_pid, ready_id); ready_port->in_fd = -1; -- cgit From 55296e6ff2613a0b2ec588beaf01620b2679c3d1 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 30 Nov 2020 23:30:20 +0300 Subject: Node.js: removing unnecessary warnings. Warnings changed for debug messages. --- src/nodejs/unit-http/unit.cpp | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index c5bca49a..b166707e 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -998,33 +998,21 @@ Unit::websocket_set_sock(napi_env env, napi_callback_info info) void -Unit::conn_destroy(napi_env env, void *nativeObject, void *finalize_hint) +Unit::conn_destroy(napi_env env, void *r, void *finalize_hint) { - nxt_unit_request_info_t *req; - - req = (nxt_unit_request_info_t *) nativeObject; - - nxt_unit_warn(NULL, "conn_destroy: %p", req); + nxt_unit_req_debug((nxt_unit_request_info_t *) r, "conn_destroy: %p", r); } void -Unit::sock_destroy(napi_env env, void *nativeObject, void *finalize_hint) +Unit::sock_destroy(napi_env env, void *r, void *finalize_hint) { - nxt_unit_request_info_t *req; - - req = (nxt_unit_request_info_t *) nativeObject; - - nxt_unit_warn(NULL, "sock_destroy: %p", req); + nxt_unit_req_debug((nxt_unit_request_info_t *) r, "sock_destroy: %p", r); } void -Unit::resp_destroy(napi_env env, void *nativeObject, void *finalize_hint) +Unit::resp_destroy(napi_env env, void *r, void *finalize_hint) { - nxt_unit_request_info_t *req; - - req = (nxt_unit_request_info_t *) nativeObject; - - nxt_unit_warn(NULL, "resp_destroy: %p", req); + nxt_unit_req_debug((nxt_unit_request_info_t *) r, "resp_destroy: %p", r); } -- cgit From 07789a23e9c513dba87b020fae2989a57955e8a6 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Sun, 6 Dec 2020 16:01:59 +0000 Subject: Tests: options moved to the separate class. This change is necessary to separate the logic and prevent possible circular dependency. --- test/conftest.py | 13 +++++++++---- test/test_access_log.py | 2 +- test/test_asgi_application.py | 2 +- test/test_asgi_lifespan.py | 2 +- test/test_asgi_websockets.py | 2 +- test/test_go_isolation.py | 2 +- test/test_java_application.py | 2 +- test/test_java_isolation_rootfs.py | 2 +- test/test_java_websockets.py | 2 +- test/test_node_websockets.py | 2 +- test/test_php_application.py | 2 +- test/test_php_isolation.py | 2 +- test/test_php_targets.py | 2 +- test/test_proxy.py | 2 +- test/test_proxy_chunked.py | 2 +- test/test_python_application.py | 2 +- test/test_python_isolation.py | 2 +- test/test_python_procman.py | 2 +- test/test_respawn.py | 2 +- test/test_routing.py | 2 +- test/test_ruby_isolation.py | 2 +- test/test_share_fallback.py | 2 +- test/test_static.py | 2 +- test/test_tls.py | 2 +- test/test_upstreams_rr.py | 2 +- test/unit/applications/lang/go.py | 2 +- test/unit/applications/lang/java.py | 2 +- test/unit/applications/lang/node.py | 2 +- test/unit/applications/lang/perl.py | 2 +- test/unit/applications/lang/php.py | 2 +- test/unit/applications/lang/python.py | 2 +- test/unit/applications/lang/ruby.py | 2 +- test/unit/applications/proto.py | 2 +- test/unit/applications/tls.py | 2 +- test/unit/control.py | 2 +- test/unit/feature/isolation.py | 2 +- test/unit/http.py | 2 +- test/unit/main.py | 2 +- test/unit/option.py | 16 ++++++++++++++++ 39 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 test/unit/option.py diff --git a/test/conftest.py b/test/conftest.py index 3edc471d..8ca8c1d3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -17,6 +17,7 @@ import pytest from unit.check.go import check_go from unit.check.node import check_node from unit.check.tls import check_openssl +from unit.option import option def pytest_addoption(parser): @@ -48,12 +49,14 @@ def pytest_addoption(parser): unit_instance = {} _processes = [] -option = None - def pytest_configure(config): - global option - option = config.option + option.config = config.option + + option.detailed = config.option.detailed + option.print_log = config.option.print_log + option.save_log = config.option.save_log + option.unsafe = config.option.unsafe option.generated_tests = {} option.current_dir = os.path.abspath( @@ -162,6 +165,8 @@ def pytest_sessionstart(session): unit_stop() + _check_alerts() + shutil.rmtree(unit_instance['temp_dir']) diff --git a/test/test_access_log.py b/test/test_access_log.py index 511ce6c5..0a0a6633 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -2,9 +2,9 @@ import time import pytest -from conftest import option from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestAccessLog(TestApplicationPython): diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index e90d78bc..80b64029 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -4,9 +4,9 @@ from distutils.version import LooseVersion import pytest -from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestASGIApplication(TestApplicationPython): diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index 3f29c7e7..409634bb 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -3,10 +3,10 @@ from distutils.version import LooseVersion import pytest -from conftest import option from conftest import public_dir from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestASGILifespan(TestApplicationPython): diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 54984526..766e53ed 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -4,10 +4,10 @@ from distutils.version import LooseVersion import pytest -from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython from unit.applications.websockets import TestApplicationWebsocket +from unit.option import option class TestASGIWebsockets(TestApplicationPython): diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index 8c4a6b9c..ac12c8ca 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -5,11 +5,11 @@ import shutil import pytest -from conftest import option from conftest import unit_run from conftest import unit_stop from unit.applications.lang.go import TestApplicationGo from unit.feature.isolation import TestFeatureIsolation +from unit.option import option class TestGoIsolation(TestApplicationGo): prerequisites = {'modules': {'go': 'any'}, 'features': ['isolation']} diff --git a/test/test_java_application.py b/test/test_java_application.py index 41345e87..04210ebf 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -3,10 +3,10 @@ import os import re import time -from conftest import option from conftest import public_dir from conftest import skip_alert from unit.applications.lang.java import TestApplicationJava +from unit.option import option class TestJavaApplication(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} diff --git a/test/test_java_isolation_rootfs.py b/test/test_java_isolation_rootfs.py index 02d35a62..a401e23b 100644 --- a/test/test_java_isolation_rootfs.py +++ b/test/test_java_isolation_rootfs.py @@ -3,8 +3,8 @@ import subprocess import pytest -from conftest import option from unit.applications.lang.java import TestApplicationJava +from unit.option import option class TestJavaIsolationRootfs(TestApplicationJava): diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 7586d4aa..5739e236 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -3,10 +3,10 @@ import time import pytest -from conftest import option from conftest import skip_alert from unit.applications.lang.java import TestApplicationJava from unit.applications.websockets import TestApplicationWebsocket +from unit.option import option class TestJavaWebsockets(TestApplicationJava): diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 7b65b5c1..84eb4c42 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -3,10 +3,10 @@ import time import pytest -from conftest import option from conftest import skip_alert from unit.applications.lang.node import TestApplicationNode from unit.applications.websockets import TestApplicationWebsocket +from unit.option import option class TestNodeWebsockets(TestApplicationNode): diff --git a/test/test_php_application.py b/test/test_php_application.py index 578de0b7..463ec35b 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -5,9 +5,9 @@ import time import pytest -from conftest import option from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP +from unit.option import option class TestPHPApplication(TestApplicationPHP): prerequisites = {'modules': {'php': 'all'}} diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index cc660e04..1d0b0614 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -2,11 +2,11 @@ import shutil import pytest -from conftest import option from conftest import unit_run from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP from unit.feature.isolation import TestFeatureIsolation +from unit.option import option class TestPHPIsolation(TestApplicationPHP): diff --git a/test/test_php_targets.py b/test/test_php_targets.py index e64cd6b6..76326131 100644 --- a/test/test_php_targets.py +++ b/test/test_php_targets.py @@ -1,5 +1,5 @@ -from conftest import option from unit.applications.lang.php import TestApplicationPHP +from unit.option import option class TestPHPTargets(TestApplicationPHP): diff --git a/test/test_proxy.py b/test/test_proxy.py index be3e93fd..90b6b657 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -4,11 +4,11 @@ import time import pytest -from conftest import option from conftest import run_process from conftest import skip_alert from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestProxy(TestApplicationPython): diff --git a/test/test_proxy_chunked.py b/test/test_proxy_chunked.py index ae2228fa..5529b397 100644 --- a/test/test_proxy_chunked.py +++ b/test/test_proxy_chunked.py @@ -3,10 +3,10 @@ import select import socket import time -from conftest import option from conftest import run_process from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestProxyChunked(TestApplicationPython): diff --git a/test/test_python_application.py b/test/test_python_application.py index 83b0c8f4..b28e8c11 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -5,10 +5,10 @@ import time import pytest -from conftest import option from conftest import skip_alert from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestPythonApplication(TestApplicationPython): diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 1a157528..66ff2f16 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -2,11 +2,11 @@ import shutil import pytest -from conftest import option from conftest import unit_run from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.feature.isolation import TestFeatureIsolation +from unit.option import option class TestPythonIsolation(TestApplicationPython): diff --git a/test/test_python_procman.py b/test/test_python_procman.py index ff914fc8..42a197a7 100644 --- a/test/test_python_procman.py +++ b/test/test_python_procman.py @@ -4,8 +4,8 @@ import time import pytest -from conftest import option from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestPythonProcman(TestApplicationPython): diff --git a/test/test_respawn.py b/test/test_respawn.py index 09a806d4..fbaad666 100644 --- a/test/test_respawn.py +++ b/test/test_respawn.py @@ -2,9 +2,9 @@ import re import subprocess import time -from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestRespawn(TestApplicationPython): diff --git a/test/test_routing.py b/test/test_routing.py index 83852273..a49075ae 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import pytest -from conftest import option from conftest import skip_alert from unit.applications.proto import TestApplicationProto +from unit.option import option class TestRouting(TestApplicationProto): diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index 69e25de9..56fccbd0 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -2,11 +2,11 @@ import shutil import pytest -from conftest import option from conftest import unit_run from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby from unit.feature.isolation import TestFeatureIsolation +from unit.option import option class TestRubyIsolation(TestApplicationRuby): diff --git a/test/test_share_fallback.py b/test/test_share_fallback.py index 462da9de..151d3ae5 100644 --- a/test/test_share_fallback.py +++ b/test/test_share_fallback.py @@ -1,8 +1,8 @@ import os -from conftest import option from conftest import skip_alert from unit.applications.proto import TestApplicationProto +from unit.option import option class TestStatic(TestApplicationProto): diff --git a/test/test_static.py b/test/test_static.py index a65928ca..1df0c902 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -3,9 +3,9 @@ import socket import pytest -from conftest import option from conftest import waitforfiles from unit.applications.proto import TestApplicationProto +from unit.option import option class TestStatic(TestApplicationProto): diff --git a/test/test_tls.py b/test/test_tls.py index 4cf8d22c..32b97895 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -5,9 +5,9 @@ import subprocess import pytest -from conftest import option from conftest import skip_alert from unit.applications.tls import TestApplicationTLS +from unit.option import option class TestTLS(TestApplicationTLS): diff --git a/test/test_upstreams_rr.py b/test/test_upstreams_rr.py index c20d6054..163eb646 100644 --- a/test/test_upstreams_rr.py +++ b/test/test_upstreams_rr.py @@ -1,8 +1,8 @@ import os import re -from conftest import option from unit.applications.lang.python import TestApplicationPython +from unit.option import option class TestUpstreamsRR(TestApplicationPython): diff --git a/test/unit/applications/lang/go.py b/test/unit/applications/lang/go.py index 866dec47..70f9d58c 100644 --- a/test/unit/applications/lang/go.py +++ b/test/unit/applications/lang/go.py @@ -1,8 +1,8 @@ import os import subprocess -from conftest import option from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationGo(TestApplicationProto): diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 0ff85187..b2e17f23 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -4,8 +4,8 @@ import shutil import subprocess import pytest -from conftest import option from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationJava(TestApplicationProto): diff --git a/test/unit/applications/lang/node.py b/test/unit/applications/lang/node.py index 98fd9ffc..baccef7e 100644 --- a/test/unit/applications/lang/node.py +++ b/test/unit/applications/lang/node.py @@ -1,9 +1,9 @@ import shutil from urllib.parse import quote -from conftest import option from conftest import public_dir from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationNode(TestApplicationProto): diff --git a/test/unit/applications/lang/perl.py b/test/unit/applications/lang/perl.py index 9dc24ace..58b867f0 100644 --- a/test/unit/applications/lang/perl.py +++ b/test/unit/applications/lang/perl.py @@ -1,5 +1,5 @@ -from conftest import option from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationPerl(TestApplicationProto): diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index 3dbb32f5..90c0078c 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -1,8 +1,8 @@ -from conftest import option import os import shutil from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationPHP(TestApplicationProto): diff --git a/test/unit/applications/lang/python.py b/test/unit/applications/lang/python.py index 792a86fa..287d23f0 100644 --- a/test/unit/applications/lang/python.py +++ b/test/unit/applications/lang/python.py @@ -3,8 +3,8 @@ import shutil from urllib.parse import quote import pytest -from conftest import option from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationPython(TestApplicationProto): diff --git a/test/unit/applications/lang/ruby.py b/test/unit/applications/lang/ruby.py index 82d66e65..02644584 100644 --- a/test/unit/applications/lang/ruby.py +++ b/test/unit/applications/lang/ruby.py @@ -1,5 +1,5 @@ -from conftest import option from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationRuby(TestApplicationProto): diff --git a/test/unit/applications/proto.py b/test/unit/applications/proto.py index 6e760c70..af05d071 100644 --- a/test/unit/applications/proto.py +++ b/test/unit/applications/proto.py @@ -2,8 +2,8 @@ import os import re import time -from conftest import option from unit.control import TestControl +from unit.option import option class TestApplicationProto(TestControl): diff --git a/test/unit/applications/tls.py b/test/unit/applications/tls.py index fb1b112c..b0cd5abb 100644 --- a/test/unit/applications/tls.py +++ b/test/unit/applications/tls.py @@ -2,8 +2,8 @@ import os import ssl import subprocess -from conftest import option from unit.applications.proto import TestApplicationProto +from unit.option import option class TestApplicationTLS(TestApplicationProto): diff --git a/test/unit/control.py b/test/unit/control.py index f05aa827..3008a64b 100644 --- a/test/unit/control.py +++ b/test/unit/control.py @@ -1,7 +1,7 @@ import json -from conftest import option from unit.http import TestHTTP +from unit.option import option def args_handler(conf_func): diff --git a/test/unit/feature/isolation.py b/test/unit/feature/isolation.py index 7877c03a..d8f68919 100644 --- a/test/unit/feature/isolation.py +++ b/test/unit/feature/isolation.py @@ -4,7 +4,7 @@ from unit.applications.lang.go import TestApplicationGo from unit.applications.lang.java import TestApplicationJava from unit.applications.lang.node import TestApplicationNode from unit.applications.proto import TestApplicationProto -from conftest import option +from unit.option import option class TestFeatureIsolation(TestApplicationProto): diff --git a/test/unit/http.py b/test/unit/http.py index 8d964978..ae74eac3 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -7,8 +7,8 @@ import select import socket import pytest -from conftest import option from unit.main import TestUnit +from unit.option import option class TestHTTP(TestUnit): diff --git a/test/unit/main.py b/test/unit/main.py index 488b3f4d..fce6a322 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -1,5 +1,5 @@ import pytest -from conftest import option +from unit.option import option class TestUnit(): diff --git a/test/unit/option.py b/test/unit/option.py new file mode 100644 index 00000000..677d806e --- /dev/null +++ b/test/unit/option.py @@ -0,0 +1,16 @@ +class Options(): + _options = { + 'skip_alerts': [], + 'skip_sanitizer': False, + } + + def __setattr__(self, name, value): + Options._options[name] = value + + def __getattr__(self, name): + if name in Options._options: + return Options._options[name] + + raise AttributeError + +option = Options() -- cgit From db42527b1b2656141af0d8280e59e23be6af67d6 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 7 Dec 2020 17:56:18 +0300 Subject: Node.js: avoided use of request struct for debug logging. This fixes a crash on exit of Node.js application. The crash reproduced on Ubuntu 20.10 with Node.js v15.1.0. Tests 'test_node_websockets_two_clients' and 'test_node_websockets_7_13_1__7_13_2'. The reason of the crash is using request struct which was already freed. The issue was introduced in 5be509fda29e. --- src/nodejs/unit-http/unit.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index b166707e..0e049865 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -1000,19 +1000,19 @@ Unit::websocket_set_sock(napi_env env, napi_callback_info info) void Unit::conn_destroy(napi_env env, void *r, void *finalize_hint) { - nxt_unit_req_debug((nxt_unit_request_info_t *) r, "conn_destroy: %p", r); + nxt_unit_req_debug(NULL, "conn_destroy: %p", r); } void Unit::sock_destroy(napi_env env, void *r, void *finalize_hint) { - nxt_unit_req_debug((nxt_unit_request_info_t *) r, "sock_destroy: %p", r); + nxt_unit_req_debug(NULL, "sock_destroy: %p", r); } void Unit::resp_destroy(napi_env env, void *r, void *finalize_hint) { - nxt_unit_req_debug((nxt_unit_request_info_t *) r, "resp_destroy: %p", r); + nxt_unit_req_debug(NULL, "resp_destroy: %p", r); } -- cgit From bda76b04e9aec295c7fa11c7eacb6e11c4bf67f4 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Mon, 7 Dec 2020 18:50:56 +0300 Subject: HTTP: fixed status line format for unknown status codes. According to Section #3.1.2 of RFC 7230, after the status code there must be a space even if the reason phrase is empty. Also, only 3 digits allowed. This closes #507 issue on GitHub. --- docs/changes.xml | 6 ++++++ src/nxt_h1proto.c | 37 ++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index 39c270ad..7b90f4a3 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -9,6 +9,12 @@ date="" time="18:00:00 +0300" packager="Andrei Belov <defan@nginx.com>"> + + +invalid HTTP responses were generated for some unusual status codes. + + + diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index dccbe56c..6aef264c 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -1151,19 +1151,19 @@ static const nxt_str_t nxt_http_client_error[] = { nxt_string("HTTP/1.1 415 Unsupported Media Type\r\n"), nxt_string("HTTP/1.1 416 Range Not Satisfiable\r\n"), nxt_string("HTTP/1.1 417 Expectation Failed\r\n"), - nxt_string("HTTP/1.1 418\r\n"), - nxt_string("HTTP/1.1 419\r\n"), - nxt_string("HTTP/1.1 420\r\n"), - nxt_string("HTTP/1.1 421\r\n"), - nxt_string("HTTP/1.1 422\r\n"), - nxt_string("HTTP/1.1 423\r\n"), - nxt_string("HTTP/1.1 424\r\n"), - nxt_string("HTTP/1.1 425\r\n"), + nxt_string("HTTP/1.1 418 I'm a teapot\r\n"), + nxt_string("HTTP/1.1 419 \r\n"), + nxt_string("HTTP/1.1 420 \r\n"), + nxt_string("HTTP/1.1 421 Misdirected Request\r\n"), + nxt_string("HTTP/1.1 422 Unprocessable Entity\r\n"), + nxt_string("HTTP/1.1 423 Locked\r\n"), + nxt_string("HTTP/1.1 424 Failed Dependency\r\n"), + nxt_string("HTTP/1.1 425 \r\n"), nxt_string("HTTP/1.1 426 Upgrade Required\r\n"), - nxt_string("HTTP/1.1 427\r\n"), - nxt_string("HTTP/1.1 428\r\n"), - nxt_string("HTTP/1.1 429\r\n"), - nxt_string("HTTP/1.1 430\r\n"), + nxt_string("HTTP/1.1 427 \r\n"), + nxt_string("HTTP/1.1 428 \r\n"), + nxt_string("HTTP/1.1 429 \r\n"), + nxt_string("HTTP/1.1 430 \r\n"), nxt_string("HTTP/1.1 431 Request Header Fields Too Large\r\n"), }; @@ -1190,7 +1190,7 @@ static const nxt_str_t nxt_http_server_error[] = { }; -#define UNKNOWN_STATUS_LENGTH nxt_length("HTTP/1.1 65536\r\n") +#define UNKNOWN_STATUS_LENGTH nxt_length("HTTP/1.1 999 \r\n") static void nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, @@ -1248,13 +1248,16 @@ nxt_h1p_request_header_send(nxt_task_t *task, nxt_http_request_t *r, { status = &nxt_http_server_error[n - NXT_HTTP_INTERNAL_SERVER_ERROR]; - } else { - p = nxt_sprintf(buf, buf + UNKNOWN_STATUS_LENGTH, - "HTTP/1.1 %03d\r\n", n); + } else if (n <= NXT_HTTP_STATUS_MAX) { + (void) nxt_sprintf(buf, buf + UNKNOWN_STATUS_LENGTH, + "HTTP/1.1 %03d \r\n", n); - unknown_status.length = p - buf; + unknown_status.length = UNKNOWN_STATUS_LENGTH; unknown_status.start = buf; status = &unknown_status; + + } else { + status = &nxt_http_server_error[0]; } size = status->length; -- cgit From 2348229dc7656f36a7915d85af56aae9ed9fb120 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Tue, 8 Dec 2020 01:59:46 +0300 Subject: PHP: populating PHP_AUTH_* server variables. This closes #498 issue on GitHub. --- docs/changes.xml | 7 +++++++ src/nxt_h1proto.c | 2 ++ src/nxt_http.h | 1 + src/nxt_php_sapi.c | 11 +++++++++++ src/nxt_router.c | 4 ++++ src/nxt_unit_request.h | 1 + 6 files changed, 26 insertions(+) diff --git a/docs/changes.xml b/docs/changes.xml index 7b90f4a3..bec07ec2 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -15,6 +15,13 @@ invalid HTTP responses were generated for some unusual status codes. + + +PHP_AUTH_USER, PHP_AUTH_PW, and PHP_AUTH_DIGEST server variables were missing +in PHP module. + + + diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 6aef264c..d3da6942 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -174,6 +174,8 @@ static nxt_http_field_proc_t nxt_h1p_fields[] = { { nxt_string("Content-Type"), &nxt_http_request_field, offsetof(nxt_http_request_t, content_type) }, { nxt_string("Content-Length"), &nxt_http_request_content_length, 0 }, + { nxt_string("Authorization"), &nxt_http_request_field, + offsetof(nxt_http_request_t, authorization) }, }; diff --git a/src/nxt_http.h b/src/nxt_http.h index 1418be95..e30bfeb4 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -156,6 +156,7 @@ struct nxt_http_request_s { nxt_http_field_t *cookie; nxt_http_field_t *referer; nxt_http_field_t *user_agent; + nxt_http_field_t *authorization; nxt_off_t content_length_n; nxt_sockaddr_t *remote; diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index d2fbdd27..369e7f32 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -1038,6 +1038,17 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) ctx->cookie = nxt_unit_sptr_get(&f->value); } + if (r->authorization_field != NXT_UNIT_NONE_FIELD) { + f = r->fields + r->authorization_field; + + php_handle_auth_data(nxt_unit_sptr_get(&f->value)); + + } else { + SG(request_info).auth_digest = NULL; + SG(request_info).auth_user = NULL; + SG(request_info).auth_password = NULL; + } + SG(sapi_headers).http_response_code = 200; SG(request_info).path_translated = NULL; diff --git a/src/nxt_router.c b/src/nxt_router.c index 9dd5c30e..871602e4 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5169,6 +5169,7 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, req->content_length_field = NXT_UNIT_NONE_FIELD; req->content_type_field = NXT_UNIT_NONE_FIELD; req->cookie_field = NXT_UNIT_NONE_FIELD; + req->authorization_field = NXT_UNIT_NONE_FIELD; dst_field = req->fields; @@ -5193,6 +5194,9 @@ nxt_router_prepare_msg(nxt_task_t *task, nxt_http_request_t *r, } else if (field == r->cookie) { req->cookie_field = dst_field - req->fields; + + } else if (field == r->authorization) { + req->authorization_field = dst_field - req->fields; } nxt_debug(task, "add field 0x%04Xd, %d, %d, %p : %d %p", diff --git a/src/nxt_unit_request.h b/src/nxt_unit_request.h index fede00d2..5dbf648d 100644 --- a/src/nxt_unit_request.h +++ b/src/nxt_unit_request.h @@ -31,6 +31,7 @@ struct nxt_unit_request_s { uint32_t content_length_field; uint32_t content_type_field; uint32_t cookie_field; + uint32_t authorization_field; uint64_t content_length; -- cgit From 12a0d259a50fd6feef46aeffa0e46e193ce6bb40 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 8 Dec 2020 13:17:21 +0300 Subject: Docs: special handling for empty "date" and "time" XML attributes. --- docs/Makefile | 9 +++++++++ docs/changes.xml | 2 +- docs/changes.xsls | 47 +++++++++++++++++++++++++++++++++++++---------- docs/changes.xslt | 47 +++++++++++++++++++++++++++++++++++++---------- 4 files changed, 84 insertions(+), 21 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index aa8aeb9b..db63eec4 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -14,6 +14,9 @@ PACKAGES= unit \ unit-ruby \ unit-jsc-common unit-jsc8 unit-jsc10 unit-jsc11 +CURDATE:=$(shell date +"%Y-%m-%d") +CURTIME:=$(shell date +"%H:%M:%S %z") + all: changes changelogs @@ -31,6 +34,8 @@ $(DEST)/CHANGES: changes.dtd \ xmllint --noout --valid changes.xml xsltproc --stringparam format generic \ + --stringparam curdate '$(CURDATE)' \ + --stringparam curtime '$(CURTIME)' \ -o $@ changes.xslt changes.xml $(DEST)/%.rpm-changelog: changes.dtd \ @@ -40,6 +45,8 @@ $(DEST)/%.rpm-changelog: changes.dtd \ mkdir -p $(DEST) xmllint --noout --valid changes.xml xsltproc --stringparam pkgname $* --stringparam format rpm \ + --stringparam curdate '$(CURDATE)' \ + --stringparam curtime '$(CURTIME)' \ -o $@ changes.xslt changes.xml $(DEST)/%.deb-changelog: changes.dtd \ @@ -49,6 +56,8 @@ $(DEST)/%.deb-changelog: changes.dtd \ mkdir -p $(DEST) xmllint --noout --valid changes.xml xsltproc --stringparam pkgname $* --stringparam format deb \ + --stringparam curdate '$(CURDATE)' \ + --stringparam curtime '$(CURTIME)' \ -o $@ changes.xslt changes.xml changes.xslt: changes.xsls diff --git a/docs/changes.xml b/docs/changes.xml index bec07ec2..11032a02 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -6,7 +6,7 @@ diff --git a/docs/changes.xsls b/docs/changes.xsls index 2f3bcc46..9ff6a7e2 100644 --- a/docs/changes.xsls +++ b/docs/changes.xsls @@ -5,6 +5,8 @@ X:output method="text"; X:param format="'generic'"; X:param pkgname="'unit'"; X:param configuration="'change_log_conf.xml'"; +X:param curdate; +X:param curtime; X:var conf = "document($configuration)/configuration"; @@ -58,8 +60,10 @@ X:template = "change_log" { !! "changes"; } X:template = "changes" { - X:var pday = { !padded_day(date="@date") } - X:var dow = { !day_of_week(date="@date") } + X:var date_ = { !getdate(date="@date", curdate="$curdate") } + X:var time_ = { !gettime(time="@time", curtime="$curtime") } + X:var pday = { !padded_day(date="$date_") } + X:var dow = { !day_of_week(date="$date_") } X:var apply = { !string_in_list(list="@apply", string="$pkgname") } X:var pkgname_ = { !beautify(pkgname="$pkgname") } @@ -77,17 +81,16 @@ X:template = "changes" { ' '), 1, $conf/changes/length)} - !{substring(@date, 9, 2)} - !{$conf/changes/month[number(substring(current()/@date, - 6, 2))]} - !{substring(@date, 1, 4)} + !{substring($date_, 9, 2)} + !{$conf/changes/month[number(substring($date_, 6, 2))]} + !{substring($date_, 1, 4)} } X:if "$format='rpm'" { !{concat('* ', $conf/changes/day[number($dow)], - $conf/changes/month[number(substring(current()/@date, 6, 2))], + $conf/changes/month[number(substring($date_, 6, 2))], $pday, ' ', - substring(@date, 1, 4), ' ', @packager, ' - ', + substring($date_, 1, 4), ' ', @packager, ' - ', @ver, '-', @rev, '%{?dist}.ngx')} } @@ -108,8 +111,8 @@ X:template = "changes" { !{concat(' -- ', @packager, ' ', $conf/changes/day[number($dow)], ', ', $pday, - $conf/changes/month[number(substring(current()/@date, 6, 2))], - substring(@date, 1, 4), ' ', @time)} + $conf/changes/month[number(substring($date_, 6, 2))], + substring($date_, 1, 4), ' ', $time_)} X:text { } X:text { } @@ -254,6 +257,30 @@ X:template beautify(pkgname) { } +X:template getdate(date, curdate) { + X:choose { + X:when "$date=''" { + !{$curdate} + } + X:otherwise { + !{$date} + } + } +} + + +X:template gettime(time, curtime) { + X:choose { + X:when "$time=''" { + !{$curtime} + } + X:otherwise { + !{$time} + } + } +} + + X:template = "at" {@} X:template = "br" { !{$br} } X:template = "nobr" { !{translate(., ' ', ' ')} } diff --git a/docs/changes.xslt b/docs/changes.xslt index 032d5c37..08f2f800 100644 --- a/docs/changes.xslt +++ b/docs/changes.xslt @@ -6,6 +6,8 @@ + + @@ -59,8 +61,10 @@ - - + + + + @@ -78,17 +82,16 @@ ' '), 1, $conf/changes/length)"/> - - - + + + @@ -109,8 +112,8 @@ + $conf/changes/month[number(substring($date_, 6, 2))], + substring($date_, 1, 4), ' ', $time_)"/> @@ -246,6 +249,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @ -- cgit From 8f916285639d7f9aac9ef03cace5e4dcbcca70cd Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 8 Dec 2020 14:37:25 +0000 Subject: Tests: utils module introduced. --- test/conftest.py | 51 ++----------------------------------- test/test_asgi_lifespan.py | 1 - test/test_java_application.py | 2 +- test/test_node_application.py | 2 +- test/test_proxy.py | 2 +- test/test_proxy_chunked.py | 2 +- test/test_static.py | 2 +- test/test_usr1.py | 2 +- test/unit/applications/lang/node.py | 2 +- test/unit/utils.py | 50 ++++++++++++++++++++++++++++++++++++ 10 files changed, 59 insertions(+), 57 deletions(-) create mode 100644 test/unit/utils.py diff --git a/test/conftest.py b/test/conftest.py index 8ca8c1d3..982e43c9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -18,6 +18,8 @@ from unit.check.go import check_go from unit.check.node import check_node from unit.check.tls import check_openssl from unit.option import option +from unit.utils import public_dir +from unit.utils import waitforfiles def pytest_addoption(parser): @@ -298,34 +300,6 @@ def unit_stop(): p.kill() return 'Could not terminate unit' -def public_dir(path): - os.chmod(path, 0o777) - - for root, dirs, files in os.walk(path): - for d in dirs: - os.chmod(os.path.join(root, d), 0o777) - for f in files: - os.chmod(os.path.join(root, f), 0o777) - -def waitforfiles(*files): - for i in range(50): - wait = False - ret = False - - for f in files: - if not os.path.exists(f): - wait = True - break - - if wait: - time.sleep(0.1) - - else: - ret = True - break - - return ret - def _check_alerts(path=None): @@ -404,27 +378,6 @@ def stop_processes(): return 'Fail to stop process(es)' -def waitforsocket(port): - ret = False - - for i in range(50): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('127.0.0.1', port)) - ret = True - break - - except KeyboardInterrupt: - raise - - except: - sock.close() - time.sleep(0.1) - - sock.close() - - assert ret, 'socket connected' - @pytest.fixture def temp_dir(request): return unit_instance['temp_dir'] diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index 409634bb..43286e22 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -3,7 +3,6 @@ from distutils.version import LooseVersion import pytest -from conftest import public_dir from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_java_application.py b/test/test_java_application.py index 04210ebf..572e18e8 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -3,10 +3,10 @@ import os import re import time -from conftest import public_dir from conftest import skip_alert from unit.applications.lang.java import TestApplicationJava from unit.option import option +from unit.utils import public_dir class TestJavaApplication(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} diff --git a/test/test_node_application.py b/test/test_node_application.py index c8c3a444..f13a01e1 100644 --- a/test/test_node_application.py +++ b/test/test_node_application.py @@ -2,8 +2,8 @@ import re import pytest -from conftest import waitforfiles from unit.applications.lang.node import TestApplicationNode +from unit.utils import waitforfiles class TestNodeApplication(TestApplicationNode): diff --git a/test/test_proxy.py b/test/test_proxy.py index 90b6b657..975a76a0 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -6,9 +6,9 @@ import pytest from conftest import run_process from conftest import skip_alert -from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython from unit.option import option +from unit.utils import waitforsocket class TestProxy(TestApplicationPython): diff --git a/test/test_proxy_chunked.py b/test/test_proxy_chunked.py index 5529b397..73d94332 100644 --- a/test/test_proxy_chunked.py +++ b/test/test_proxy_chunked.py @@ -4,9 +4,9 @@ import socket import time from conftest import run_process -from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython from unit.option import option +from unit.utils import waitforsocket class TestProxyChunked(TestApplicationPython): diff --git a/test/test_static.py b/test/test_static.py index 1df0c902..4591aa03 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -3,9 +3,9 @@ import socket import pytest -from conftest import waitforfiles from unit.applications.proto import TestApplicationProto from unit.option import option +from unit.utils import waitforfiles class TestStatic(TestApplicationProto): diff --git a/test/test_usr1.py b/test/test_usr1.py index 3e44e4c5..44f19d23 100644 --- a/test/test_usr1.py +++ b/test/test_usr1.py @@ -2,8 +2,8 @@ import os from subprocess import call from conftest import unit_stop -from conftest import waitforfiles from unit.applications.lang.python import TestApplicationPython +from unit.utils import waitforfiles class TestUSR1(TestApplicationPython): diff --git a/test/unit/applications/lang/node.py b/test/unit/applications/lang/node.py index baccef7e..cc6d06ef 100644 --- a/test/unit/applications/lang/node.py +++ b/test/unit/applications/lang/node.py @@ -1,9 +1,9 @@ import shutil from urllib.parse import quote -from conftest import public_dir from unit.applications.proto import TestApplicationProto from unit.option import option +from unit.utils import public_dir class TestApplicationNode(TestApplicationProto): diff --git a/test/unit/utils.py b/test/unit/utils.py new file mode 100644 index 00000000..f24e9728 --- /dev/null +++ b/test/unit/utils.py @@ -0,0 +1,50 @@ +import os +import socket +import time + +import pytest + + +def public_dir(path): + os.chmod(path, 0o777) + + for root, dirs, files in os.walk(path): + for d in dirs: + os.chmod(os.path.join(root, d), 0o777) + for f in files: + os.chmod(os.path.join(root, f), 0o777) + + +def waitforfiles(*files): + for i in range(50): + wait = False + + for f in files: + if not os.path.exists(f): + wait = True + break + + if not wait: + return True + + time.sleep(0.1) + + return False + + +def waitforsocket(port): + for i in range(50): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.settimeout(5) + sock.connect(('127.0.0.1', port)) + return + + except ConnectionRefusedError: + time.sleep(0.1) + + except KeyboardInterrupt: + raise + + pytest.fail('Can\'t connect to the 127.0.0.1:' + port) + -- cgit From b2e767819f04153944d525ef8d97d2f3a7a9af74 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 8 Dec 2020 14:37:33 +0000 Subject: Tests: skip_alert() converted to the fixture. --- test/conftest.py | 14 +++++++++----- test/test_asgi_application.py | 3 +-- test/test_asgi_websockets.py | 4 ++-- test/test_configuration.py | 3 +-- test/test_java_application.py | 3 +-- test/test_java_websockets.py | 4 ++-- test/test_node_websockets.py | 4 ++-- test/test_perl_application.py | 3 +-- test/test_proxy.py | 3 +-- test/test_python_application.py | 7 +++---- test/test_respawn.py | 7 +++---- test/test_routing.py | 7 +++---- test/test_ruby_application.py | 3 +-- test/test_share_fallback.py | 3 +-- test/test_tls.py | 3 +-- 15 files changed, 32 insertions(+), 39 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 982e43c9..6783f16d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -74,10 +74,6 @@ def pytest_configure(config): fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, 0) -def skip_alert(*alerts): - option.skip_alerts.extend(alerts) - - def pytest_generate_tests(metafunc): cls = metafunc.cls if (not hasattr(cls, 'application_type') @@ -314,7 +310,7 @@ def _check_alerts(path=None): alerts = re.findall(r'.+\[alert\].+', log) if alerts: - print('All alerts/sanitizer errors found in log:') + print('\nAll alerts/sanitizer errors found in log:') [print(alert) for alert in alerts] found = True @@ -378,6 +374,14 @@ def stop_processes(): return 'Fail to stop process(es)' +@pytest.fixture() +def skip_alert(): + def _skip(*alerts): + option.skip_alerts.extend(alerts) + + return _skip + + @pytest.fixture def temp_dir(request): return unit_instance['temp_dir'] diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 80b64029..9f4b70a4 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -4,7 +4,6 @@ from distutils.version import LooseVersion import pytest -from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython from unit.option import option @@ -361,7 +360,7 @@ Connection: close self.get(headers=headers_delay_1) - def test_asgi_application_loading_error(self): + def test_asgi_application_loading_error(self, skip_alert): skip_alert(r'Python failed to import module "blah"') self.load('empty', module="blah") diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 766e53ed..76cd8e80 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -4,7 +4,6 @@ from distutils.version import LooseVersion import pytest -from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython from unit.applications.websockets import TestApplicationWebsocket from unit.option import option @@ -17,7 +16,8 @@ class TestASGIWebsockets(TestApplicationPython): ws = TestApplicationWebsocket() - def setup_method(self): + @pytest.fixture(autouse=True) + def setup_method_fixture(self, request, skip_alert): assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_configuration.py b/test/test_configuration.py index d1e6f000..7feb3adb 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -1,6 +1,5 @@ import pytest -from conftest import skip_alert from unit.control import TestControl @@ -337,7 +336,7 @@ class TestConfiguration(TestControl): assert 'success' in self.conf(conf) - def test_unprivileged_user_error(self, is_su): + def test_unprivileged_user_error(self, is_su, skip_alert): skip_alert(r'cannot set user "root"', r'failed to apply new conf') if is_su: pytest.skip('unprivileged tests') diff --git a/test/test_java_application.py b/test/test_java_application.py index 572e18e8..4a67f291 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -3,7 +3,6 @@ import os import re import time -from conftest import skip_alert from unit.applications.lang.java import TestApplicationJava from unit.option import option from unit.utils import public_dir @@ -11,7 +10,7 @@ from unit.utils import public_dir class TestJavaApplication(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} - def test_java_conf_error(self, temp_dir): + def test_java_conf_error(self, temp_dir, skip_alert): skip_alert( r'realpath.*failed', r'failed to apply new conf', diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 5739e236..729aa31d 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -3,7 +3,6 @@ import time import pytest -from conftest import skip_alert from unit.applications.lang.java import TestApplicationJava from unit.applications.websockets import TestApplicationWebsocket from unit.option import option @@ -14,7 +13,8 @@ class TestJavaWebsockets(TestApplicationJava): ws = TestApplicationWebsocket() - def setup_method(self): + @pytest.fixture(autouse=True) + def setup_method_fixture(self, request, skip_alert): assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 84eb4c42..d7444bf7 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -3,7 +3,6 @@ import time import pytest -from conftest import skip_alert from unit.applications.lang.node import TestApplicationNode from unit.applications.websockets import TestApplicationWebsocket from unit.option import option @@ -14,7 +13,8 @@ class TestNodeWebsockets(TestApplicationNode): ws = TestApplicationWebsocket() - def setup_method(self): + @pytest.fixture(autouse=True) + def setup_method_fixture(self, request, skip_alert): assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_perl_application.py b/test/test_perl_application.py index 78f2dd90..ad91965f 100644 --- a/test/test_perl_application.py +++ b/test/test_perl_application.py @@ -2,7 +2,6 @@ import re import pytest -from conftest import skip_alert from conftest import unit_stop from unit.applications.lang.perl import TestApplicationPerl @@ -170,7 +169,7 @@ class TestPerlApplication(TestApplicationPerl): assert self.get()['body'] == 'body\n', 'body io file' @pytest.mark.skip('not yet') - def test_perl_application_syntax_error(self): + def test_perl_application_syntax_error(self, skip_alert): skip_alert(r'PSGI: Failed to parse script') self.load('syntax_error') diff --git a/test/test_proxy.py b/test/test_proxy.py index 975a76a0..0eebab1e 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -5,7 +5,6 @@ import time import pytest from conftest import run_process -from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython from unit.option import option from unit.utils import waitforsocket @@ -482,7 +481,7 @@ Content-Length: 10 check_proxy('http://[:]:7080') check_proxy('http://[::7080') - def test_proxy_loop(self): + def test_proxy_loop(self, skip_alert): skip_alert( r'socket.*failed', r'accept.*failed', diff --git a/test/test_python_application.py b/test/test_python_application.py index b28e8c11..b7cec831 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -5,7 +5,6 @@ import time import pytest -from conftest import skip_alert from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.option import option @@ -520,13 +519,13 @@ last line: 987654321 assert self.get()['body'] == 'body\n', 'body io file' @pytest.mark.skip('not yet') - def test_python_application_syntax_error(self): + def test_python_application_syntax_error(self, skip_alert): skip_alert(r'Python failed to import module "wsgi"') self.load('syntax_error') assert self.get()['status'] == 500, 'syntax error' - def test_python_application_loading_error(self): + def test_python_application_loading_error(self, skip_alert): skip_alert(r'Python failed to import module "blah"') self.load('empty', module="blah") @@ -791,7 +790,7 @@ last line: 987654321 assert obj['UID'] == nobody_uid, 'root uid group=root' assert obj['GID'] == 0, 'root gid group=root' - def test_python_application_callable(self): + def test_python_application_callable(self, skip_alert): skip_alert(r'Python failed to get "blah" from module') self.load('callable') diff --git a/test/test_respawn.py b/test/test_respawn.py index fbaad666..eef2cb56 100644 --- a/test/test_respawn.py +++ b/test/test_respawn.py @@ -2,7 +2,6 @@ import re import subprocess import time -from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython from unit.option import option @@ -56,7 +55,7 @@ class TestRespawn(TestApplicationPython): assert len(re.findall(self.PATTERN_CONTROLLER, output)) == 1 assert len(re.findall(self.app_name, output)) == 1 - def test_respawn_router(self): + def test_respawn_router(self, skip_alert): pid = self.pid_by_name(self.PATTERN_ROUTER) self.kill_pids(pid) @@ -66,7 +65,7 @@ class TestRespawn(TestApplicationPython): self.smoke_test() - def test_respawn_controller(self): + def test_respawn_controller(self, skip_alert): pid = self.pid_by_name(self.PATTERN_CONTROLLER) self.kill_pids(pid) @@ -78,7 +77,7 @@ class TestRespawn(TestApplicationPython): self.smoke_test() - def test_respawn_application(self): + def test_respawn_application(self, skip_alert): pid = self.pid_by_name(self.app_name) self.kill_pids(pid) diff --git a/test/test_routing.py b/test/test_routing.py index a49075ae..30f25a9c 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import pytest -from conftest import skip_alert from unit.applications.proto import TestApplicationProto from unit.option import option @@ -366,7 +365,7 @@ class TestRouting(TestApplicationProto): assert self.get()['status'] == 200, 'route match absent' - def test_routes_route_action_absent(self): + def test_routes_route_action_absent(self, skip_alert): skip_alert(r'failed to apply new conf') assert 'error' in self.conf( @@ -755,7 +754,7 @@ class TestRouting(TestApplicationProto): 'routes/main' ), 'route edit configure 9' - def test_match_edit(self): + def test_match_edit(self, skip_alert): skip_alert(r'failed to apply new conf') self.route_match({"method": ["GET", "POST"]}) @@ -1352,7 +1351,7 @@ class TestRouting(TestApplicationProto): assert self.get(url='/?var2=val2')['status'] == 404, 'arr 7' assert self.get(url='/?var3=foo')['status'] == 200, 'arr 8' - def test_routes_match_arguments_invalid(self): + def test_routes_match_arguments_invalid(self, skip_alert): # TODO remove it after controller fixed skip_alert(r'failed to apply new conf') diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index e42fb97f..6a0c9c9f 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -2,7 +2,6 @@ import re import pytest -from conftest import skip_alert from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby @@ -160,7 +159,7 @@ class TestRubyApplication(TestApplicationRuby): assert self.post(body=body)['body'] == body, 'input rewind' @pytest.mark.skip('not yet') - def test_ruby_application_syntax_error(self): + def test_ruby_application_syntax_error(self, skip_alert): skip_alert( r'Failed to parse rack script', r'syntax error', diff --git a/test/test_share_fallback.py b/test/test_share_fallback.py index 151d3ae5..a02cb1a3 100644 --- a/test/test_share_fallback.py +++ b/test/test_share_fallback.py @@ -1,6 +1,5 @@ import os -from conftest import skip_alert from unit.applications.proto import TestApplicationProto from unit.option import option @@ -117,7 +116,7 @@ class TestStatic(TestApplicationProto): assert resp['status'] == 200, 'fallback proxy status' assert resp['body'] == '', 'fallback proxy' - def test_fallback_proxy_loop(self): + def test_fallback_proxy_loop(self, skip_alert): skip_alert( r'open.*/blah/index.html.*failed', r'accept.*failed', diff --git a/test/test_tls.py b/test/test_tls.py index 32b97895..5a408ee2 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -5,7 +5,6 @@ import subprocess import pytest -from conftest import skip_alert from unit.applications.tls import TestApplicationTLS from unit.option import option @@ -505,7 +504,7 @@ basicConstraints = critical,CA:TRUE""" '/certificates' ), 'remove all certificates' - def test_tls_application_respawn(self): + def test_tls_application_respawn(self, skip_alert): self.load('mirror') self.certificate() -- cgit From c7bd96b4769abb11a92a081061bc83171d5ed6ac Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Tue, 8 Dec 2020 16:38:41 +0000 Subject: Tests: fixed ruby isolation. While alternating between running priv and unpriv tests locally, it happens that unpriv tests can't bind mount or create sub directories inside directories created by root. This patch fixes this by pointing "rootfs" to temporary directory. Now the priv and unpriv test uses the same test function. --- test/test_ruby_isolation.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index 56fccbd0..8c382e5a 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -1,5 +1,6 @@ import shutil +import os import pytest from conftest import unit_run @@ -26,7 +27,7 @@ class TestRubyIsolation(TestApplicationRuby): return check if not complete_check else check() - def test_ruby_isolation_rootfs_mount_namespace(self, is_su): + def test_ruby_isolation_rootfs(self, is_su): isolation_features = option.available['features']['isolation'].keys() if not is_su: @@ -42,34 +43,22 @@ class TestRubyIsolation(TestApplicationRuby): if 'pid' not in isolation_features: pytest.skip('pid namespace is not supported') - isolation = {'rootfs': option.test_dir} + isolation = {'rootfs': option.temp_dir} if not is_su: isolation['namespaces'] = { 'mount': True, 'credential': True, - 'pid': True + 'pid': True, } - self.load('status_int', isolation=isolation) + os.mkdir(option.temp_dir + '/ruby') - assert 'success' in self.conf( - '"/ruby/status_int/config.ru"', 'applications/status_int/script', - ) - - assert 'success' in self.conf( - '"/ruby/status_int"', 'applications/status_int/working_directory', + shutil.copytree( + option.test_dir + '/ruby/status_int', + option.temp_dir + '/ruby/status_int', ) - assert self.get()['status'] == 200, 'status int' - - def test_ruby_isolation_rootfs(self, is_su): - if not is_su: - pytest.skip('requires root') - return - - isolation = {'rootfs': option.test_dir} - self.load('status_int', isolation=isolation) assert 'success' in self.conf( -- cgit From d3796d1fb7008629a8fa505481dab96efe60cbdb Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 7 Dec 2020 18:17:25 +0000 Subject: Ruby: fixed crash on thread start. Ruby threads need to be created with GVL; otherwise, an attempt to access locked resources may occur, causing a crash. The issue was occasionally reproduced on Ubuntu 18.04 with Ruby 2.5.1 while running test_ruby_application_threads. --- docs/changes.xml | 6 ++++++ src/ruby/nxt_ruby.c | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/changes.xml b/docs/changes.xml index 11032a02..a67a671b 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -22,6 +22,12 @@ in PHP module. + + +Ruby apps with configured mutlithreading could crash on start under load. + + + diff --git a/src/ruby/nxt_ruby.c b/src/ruby/nxt_ruby.c index 698d4a43..0aad887d 100644 --- a/src/ruby/nxt_ruby.c +++ b/src/ruby/nxt_ruby.c @@ -38,6 +38,7 @@ static int nxt_ruby_init_io(nxt_ruby_ctx_t *rctx); static void nxt_ruby_request_handler(nxt_unit_request_info_t *req); static void *nxt_ruby_request_handler_gvl(void *req); static int nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_ruby_thread_create_gvl(void *rctx); static VALUE nxt_ruby_thread_func(VALUE arg); static void *nxt_ruby_unit_run(void *ctx); static void nxt_ruby_ubf(void *ctx); @@ -1141,7 +1142,7 @@ nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx) rctx->ctx = ctx; - res = rb_thread_create(RUBY_METHOD_FUNC(nxt_ruby_thread_func), rctx); + res = (VALUE) rb_thread_call_with_gvl(nxt_ruby_thread_create_gvl, rctx); if (nxt_fast_path(res != Qnil)) { nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); @@ -1159,6 +1160,17 @@ nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx) } +static void * +nxt_ruby_thread_create_gvl(void *rctx) +{ + VALUE res; + + res = rb_thread_create(RUBY_METHOD_FUNC(nxt_ruby_thread_func), rctx); + + return (void *) (uintptr_t) res; +} + + static VALUE nxt_ruby_thread_func(VALUE arg) { -- cgit From 783cdc2a3d99bd9fb8d75218d679ddb571420e98 Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Wed, 2 Dec 2020 17:33:09 +0300 Subject: Docker: creating tags and pushing to AWS ECR as well. --- pkg/docker/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile index aed5b8f7..3b6c2720 100644 --- a/pkg/docker/Makefile +++ b/pkg/docker/Makefile @@ -55,13 +55,17 @@ build-%: Dockerfile.% tag-%: build-% docker tag unit:$(VERSION)-$* nginx/unit:$(VERSION)-$* + docker tag unit:$(VERSION)-$* public.ecr.aws/nginx/unit:$(VERSION)-$* push-%: tag-% docker push nginx/unit:$(VERSION)-$* + docker push public.ecr.aws/nginx/unit:$(VERSION)-$* latest: docker tag nginx/unit:$(VERSION)-full nginx/unit:latest + docker tag nginx/unit:$(VERSION)-full public.ecr.aws/nginx/unit:latest docker push nginx/unit:latest + docker push public.ecr.aws/nginx/unit:latest refresh-base: docker pull $(shell head -n 1 Dockerfile.tmpl | cut -d' ' -f 2) -- cgit From 4c846ae69308983050a55f6467c2d53e78120e0b Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 9 Dec 2020 16:15:50 +0000 Subject: Tests: isolation check moved to the pytest_sessionstart(). This change eliminates the need for some classes to run Unit one more time before running tests. --- test/conftest.py | 4 + test/test_go_isolation.py | 24 +----- test/test_php_isolation.py | 17 ---- test/test_python_isolation.py | 17 ---- test/test_python_isolation_chroot.py | 1 - test/test_ruby_isolation.py | 17 ---- test/unit/check/isolation.py | 158 ++++++++++++++++++++++++++++++++++ test/unit/feature/isolation.py | 160 ----------------------------------- test/unit/main.py | 46 +++++----- test/unit/utils.py | 12 +++ 10 files changed, 197 insertions(+), 259 deletions(-) create mode 100644 test/unit/check/isolation.py delete mode 100644 test/unit/feature/isolation.py diff --git a/test/conftest.py b/test/conftest.py index 6783f16d..07d5f059 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -17,6 +17,7 @@ import pytest from unit.check.go import check_go from unit.check.node import check_node from unit.check.tls import check_openssl +from unit.check.isolation import check_isolation from unit.option import option from unit.utils import public_dir from unit.utils import waitforfiles @@ -123,6 +124,7 @@ def pytest_sessionstart(session): option.available = {'modules': {}, 'features': {}} unit = unit_run() + option.temp_dir = unit['temp_dir'] # read unit.log @@ -161,6 +163,8 @@ def pytest_sessionstart(session): k: v for k, v in option.available['modules'].items() if v is not None } + check_isolation() + unit_stop() _check_alerts() diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index ac12c8ca..48c1b80c 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -5,31 +5,13 @@ import shutil import pytest -from conftest import unit_run -from conftest import unit_stop from unit.applications.lang.go import TestApplicationGo -from unit.feature.isolation import TestFeatureIsolation from unit.option import option +from unit.utils import getns class TestGoIsolation(TestApplicationGo): prerequisites = {'modules': {'go': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - - @classmethod - def setup_class(cls, complete_check=True): - check = super().setup_class(complete_check=False) - - unit = unit_run() - option.temp_dir = unit['temp_dir'] - - TestFeatureIsolation().check(option.available, unit['temp_dir']) - - assert unit_stop() is None - shutil.rmtree(unit['temp_dir']) - - return check if not complete_check else check() - def unpriv_creds(self): nobody_uid = pwd.getpwnam('nobody').pw_uid @@ -219,8 +201,8 @@ class TestGoIsolation(TestApplicationGo): == option.available['features']['isolation'][ns] ), ('%s match' % ns) - assert obj['NS']['MNT'] != self.isolation.getns('mnt'), 'mnt set' - assert obj['NS']['USER'] != self.isolation.getns('user'), 'user set' + assert obj['NS']['MNT'] != getns('mnt'), 'mnt set' + assert obj['NS']['USER'] != getns('user'), 'user set' def test_isolation_pid(self, is_su): if not self.isolation_key('pid'): diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index 1d0b0614..b0fea383 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -2,30 +2,13 @@ import shutil import pytest -from conftest import unit_run -from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP -from unit.feature.isolation import TestFeatureIsolation from unit.option import option class TestPHPIsolation(TestApplicationPHP): prerequisites = {'modules': {'php': 'any'}, 'features': ['isolation']} - @classmethod - def setup_class(cls, complete_check=True): - check = super().setup_class(complete_check=False) - - unit = unit_run() - option.temp_dir = unit['temp_dir'] - - TestFeatureIsolation().check(option.available, unit['temp_dir']) - - assert unit_stop() is None - shutil.rmtree(unit['temp_dir']) - - return check if not complete_check else check() - def test_php_isolation_rootfs(self, is_su, temp_dir): isolation_features = option.available['features']['isolation'].keys() diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index 66ff2f16..ad830269 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -2,30 +2,13 @@ import shutil import pytest -from conftest import unit_run -from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython -from unit.feature.isolation import TestFeatureIsolation from unit.option import option class TestPythonIsolation(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}, 'features': ['isolation']} - @classmethod - def setup_class(cls, complete_check=True): - check = super().setup_class(complete_check=False) - - unit = unit_run() - option.temp_dir = unit['temp_dir'] - - TestFeatureIsolation().check(option.available, unit['temp_dir']) - - assert unit_stop() is None - shutil.rmtree(unit['temp_dir']) - - return check if not complete_check else check() - def test_python_isolation_rootfs(self, is_su, temp_dir): isolation_features = option.available['features']['isolation'].keys() diff --git a/test/test_python_isolation_chroot.py b/test/test_python_isolation_chroot.py index 8018d5b9..7f559bcc 100644 --- a/test/test_python_isolation_chroot.py +++ b/test/test_python_isolation_chroot.py @@ -1,7 +1,6 @@ import pytest from unit.applications.lang.python import TestApplicationPython -from unit.feature.isolation import TestFeatureIsolation class TestPythonIsolation(TestApplicationPython): diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index 8c382e5a..fab428e9 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -3,30 +3,13 @@ import shutil import os import pytest -from conftest import unit_run -from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby -from unit.feature.isolation import TestFeatureIsolation from unit.option import option class TestRubyIsolation(TestApplicationRuby): prerequisites = {'modules': {'ruby': 'any'}, 'features': ['isolation']} - @classmethod - def setup_class(cls, complete_check=True): - check = super().setup_class(complete_check=False) - - unit = unit_run() - option.temp_dir = unit['temp_dir'] - - TestFeatureIsolation().check(option.available, unit['temp_dir']) - - assert unit_stop() is None - shutil.rmtree(unit['temp_dir']) - - return check if not complete_check else check() - def test_ruby_isolation_rootfs(self, is_su): isolation_features = option.available['features']['isolation'].keys() diff --git a/test/unit/check/isolation.py b/test/unit/check/isolation.py new file mode 100644 index 00000000..bb8feed1 --- /dev/null +++ b/test/unit/check/isolation.py @@ -0,0 +1,158 @@ +import json +import os + +from unit.applications.lang.go import TestApplicationGo +from unit.applications.lang.java import TestApplicationJava +from unit.applications.lang.node import TestApplicationNode +from unit.applications.proto import TestApplicationProto +from unit.http import TestHTTP +from unit.option import option +from unit.utils import getns + +allns = ['pid', 'mnt', 'ipc', 'uts', 'cgroup', 'net'] +http = TestHTTP() + +def check_isolation(): + test_conf = {"namespaces": {"credential": True}} + available = option.available + + conf = '' + if 'go' in available['modules']: + TestApplicationGo().prepare_env('empty', 'app') + + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/go/empty", + "executable": option.temp_dir + "/go/app", + "isolation": {"namespaces": {"credential": True}}, + }, + }, + } + + elif 'python' in available['modules']: + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "python", + "processes": {"spare": 0}, + "path": option.test_dir + "/python/empty", + "working_directory": option.test_dir + "/python/empty", + "module": "wsgi", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + elif 'php' in available['modules']: + conf = { + "listeners": {"*:7080": {"pass": "applications/phpinfo"}}, + "applications": { + "phpinfo": { + "type": "php", + "processes": {"spare": 0}, + "root": option.test_dir + "/php/phpinfo", + "working_directory": option.test_dir + "/php/phpinfo", + "index": "index.php", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + elif 'ruby' in available['modules']: + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "ruby", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/ruby/empty", + "script": option.test_dir + "/ruby/empty/config.ru", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + elif 'java' in available['modules']: + TestApplicationJava().prepare_env('empty') + + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "unit_jars": option.current_dir + "/build", + "type": "java", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/java/empty/", + "webapp": option.temp_dir + "/java", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + elif 'node' in available['modules']: + TestApplicationNode().prepare_env('basic') + + conf = { + "listeners": {"*:7080": {"pass": "applications/basic"}}, + "applications": { + "basic": { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.temp_dir + "/node", + "executable": "app.js", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + elif 'perl' in available['modules']: + conf = { + "listeners": {"*:7080": {"pass": "applications/body_empty"}}, + "applications": { + "body_empty": { + "type": "perl", + "processes": {"spare": 0}, + "working_directory": option.test_dir + + "/perl/body_empty", + "script": option.test_dir + "/perl/body_empty/psgi.pl", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + else: + return + + resp = http.put( + url='/config', + sock_type='unix', + addr=option.temp_dir + '/control.unit.sock', + body=json.dumps(conf), + ) + + if 'success' not in resp: + return + + userns = getns('user') + if not userns: + return + + available['features']['isolation'] = {'user': userns} + + unp_clone_path = '/proc/sys/kernel/unprivileged_userns_clone' + if os.path.exists(unp_clone_path): + with open(unp_clone_path, 'r') as f: + if str(f.read()).rstrip() == '1': + available['features']['isolation'][ + 'unprivileged_userns_clone' + ] = True + + for ns in allns: + ns_value = getns(ns) + if ns_value: + available['features']['isolation'][ns] = ns_value diff --git a/test/unit/feature/isolation.py b/test/unit/feature/isolation.py deleted file mode 100644 index d8f68919..00000000 --- a/test/unit/feature/isolation.py +++ /dev/null @@ -1,160 +0,0 @@ -import os - -from unit.applications.lang.go import TestApplicationGo -from unit.applications.lang.java import TestApplicationJava -from unit.applications.lang.node import TestApplicationNode -from unit.applications.proto import TestApplicationProto -from unit.option import option - - -class TestFeatureIsolation(TestApplicationProto): - allns = ['pid', 'mnt', 'ipc', 'uts', 'cgroup', 'net'] - - def check(self, available, temp_dir): - test_conf = {"namespaces": {"credential": True}} - - conf = '' - if 'go' in available['modules']: - TestApplicationGo().prepare_env('empty', 'app') - - conf = { - "listeners": {"*:7080": {"pass": "applications/empty"}}, - "applications": { - "empty": { - "type": "external", - "processes": {"spare": 0}, - "working_directory": option.test_dir + "/go/empty", - "executable": option.temp_dir + "/go/app", - "isolation": {"namespaces": {"credential": True}}, - }, - }, - } - - elif 'python' in available['modules']: - conf = { - "listeners": {"*:7080": {"pass": "applications/empty"}}, - "applications": { - "empty": { - "type": "python", - "processes": {"spare": 0}, - "path": option.test_dir + "/python/empty", - "working_directory": option.test_dir + "/python/empty", - "module": "wsgi", - "isolation": {"namespaces": {"credential": True}}, - } - }, - } - - elif 'php' in available['modules']: - conf = { - "listeners": {"*:7080": {"pass": "applications/phpinfo"}}, - "applications": { - "phpinfo": { - "type": "php", - "processes": {"spare": 0}, - "root": option.test_dir + "/php/phpinfo", - "working_directory": option.test_dir + "/php/phpinfo", - "index": "index.php", - "isolation": {"namespaces": {"credential": True}}, - } - }, - } - - elif 'ruby' in available['modules']: - conf = { - "listeners": {"*:7080": {"pass": "applications/empty"}}, - "applications": { - "empty": { - "type": "ruby", - "processes": {"spare": 0}, - "working_directory": option.test_dir + "/ruby/empty", - "script": option.test_dir + "/ruby/empty/config.ru", - "isolation": {"namespaces": {"credential": True}}, - } - }, - } - - elif 'java' in available['modules']: - TestApplicationJava().prepare_env('empty') - - conf = { - "listeners": {"*:7080": {"pass": "applications/empty"}}, - "applications": { - "empty": { - "unit_jars": option.current_dir + "/build", - "type": "java", - "processes": {"spare": 0}, - "working_directory": option.test_dir + "/java/empty/", - "webapp": option.temp_dir + "/java", - "isolation": {"namespaces": {"credential": True}}, - } - }, - } - - elif 'node' in available['modules']: - TestApplicationNode().prepare_env('basic') - - conf = { - "listeners": {"*:7080": {"pass": "applications/basic"}}, - "applications": { - "basic": { - "type": "external", - "processes": {"spare": 0}, - "working_directory": option.temp_dir + "/node", - "executable": "app.js", - "isolation": {"namespaces": {"credential": True}}, - } - }, - } - - elif 'perl' in available['modules']: - conf = { - "listeners": {"*:7080": {"pass": "applications/body_empty"}}, - "applications": { - "body_empty": { - "type": "perl", - "processes": {"spare": 0}, - "working_directory": option.test_dir - + "/perl/body_empty", - "script": option.test_dir + "/perl/body_empty/psgi.pl", - "isolation": {"namespaces": {"credential": True}}, - } - }, - } - - else: - return - - if 'success' not in self.conf(conf): - return - - userns = self.getns('user') - if not userns: - return - - available['features']['isolation'] = {'user': userns} - - unp_clone_path = '/proc/sys/kernel/unprivileged_userns_clone' - if os.path.exists(unp_clone_path): - with open(unp_clone_path, 'r') as f: - if str(f.read()).rstrip() == '1': - available['features']['isolation'][ - 'unprivileged_userns_clone' - ] = True - - for ns in self.allns: - ns_value = self.getns(ns) - if ns_value: - available['features']['isolation'][ns] = ns_value - - def getns(self, nstype): - # read namespace id from symlink file: - # it points to: ':[]' - # # eg.: 'pid:[4026531836]' - nspath = '/proc/self/ns/' + nstype - data = None - - if os.path.exists(nspath): - data = int(os.readlink(nspath)[len(nstype) + 2 : -1]) - - return data diff --git a/test/unit/main.py b/test/unit/main.py index fce6a322..749ff3ab 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -4,39 +4,33 @@ from unit.option import option class TestUnit(): @classmethod - def setup_class(cls, complete_check=True): - def check(): - missed = [] + def setup_class(cls): + missed = [] - # check modules + # check modules - if 'modules' in cls.prerequisites: - available_modules = list(option.available['modules'].keys()) + if 'modules' in cls.prerequisites: + available_modules = list(option.available['modules'].keys()) - for module in cls.prerequisites['modules']: - if module in available_modules: - continue + for module in cls.prerequisites['modules']: + if module in available_modules: + continue - missed.append(module) + missed.append(module) - if missed: - pytest.skip('Unit has no ' + ', '.join(missed) + ' module(s)') + if missed: + pytest.skip('Unit has no ' + ', '.join(missed) + ' module(s)') - # check features + # check features - if 'features' in cls.prerequisites: - available_features = list(option.available['features'].keys()) + if 'features' in cls.prerequisites: + available_features = list(option.available['features'].keys()) - for feature in cls.prerequisites['features']: - if feature in available_features: - continue + for feature in cls.prerequisites['features']: + if feature in available_features: + continue - missed.append(feature) + missed.append(feature) - if missed: - pytest.skip(', '.join(missed) + ' feature(s) not supported') - - if complete_check: - check() - else: - return check + if missed: + pytest.skip(', '.join(missed) + ' feature(s) not supported') diff --git a/test/unit/utils.py b/test/unit/utils.py index f24e9728..1307a4f6 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -48,3 +48,15 @@ def waitforsocket(port): pytest.fail('Can\'t connect to the 127.0.0.1:' + port) + +def getns(nstype): + # read namespace id from symlink file: + # it points to: ':[]' + # # eg.: 'pid:[4026531836]' + nspath = '/proc/self/ns/' + nstype + data = None + + if os.path.exists(nspath): + data = int(os.readlink(nspath)[len(nstype) + 2 : -1]) + + return data -- cgit From 7be62c3c213c3da1da1a45c8db16192eb0ed14d8 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 9 Dec 2020 16:17:46 +0000 Subject: Tests: TestUnit class removed. Prerequisite checks moved to the fixture in conftest.py. --- test/conftest.py | 34 ++++++++++++++++++++++++++++++++++ test/unit/http.py | 3 +-- test/unit/main.py | 36 ------------------------------------ 3 files changed, 35 insertions(+), 38 deletions(-) delete mode 100644 test/unit/main.py diff --git a/test/conftest.py b/test/conftest.py index 07d5f059..5c8da65f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -184,6 +184,40 @@ def pytest_runtest_makereport(item, call): setattr(item, "rep_" + rep.when, rep) +@pytest.fixture(scope='class', autouse=True) +def check_prerequisites(request): + cls = request.cls + missed = [] + + # check modules + + if 'modules' in cls.prerequisites: + available_modules = list(option.available['modules'].keys()) + + for module in cls.prerequisites['modules']: + if module in available_modules: + continue + + missed.append(module) + + if missed: + pytest.skip('Unit has no ' + ', '.join(missed) + ' module(s)') + + # check features + + if 'features' in cls.prerequisites: + available_features = list(option.available['features'].keys()) + + for feature in cls.prerequisites['features']: + if feature in available_features: + continue + + missed.append(feature) + + if missed: + pytest.skip(', '.join(missed) + ' feature(s) not supported') + + @pytest.fixture(autouse=True) def run(request): unit = unit_run() diff --git a/test/unit/http.py b/test/unit/http.py index ae74eac3..57e6ed3a 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -7,11 +7,10 @@ import select import socket import pytest -from unit.main import TestUnit from unit.option import option -class TestHTTP(TestUnit): +class TestHTTP(): def http(self, start_str, **kwargs): sock_type = kwargs.get('sock_type', 'ipv4') port = kwargs.get('port', 7080) diff --git a/test/unit/main.py b/test/unit/main.py deleted file mode 100644 index 749ff3ab..00000000 --- a/test/unit/main.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -from unit.option import option - - -class TestUnit(): - @classmethod - def setup_class(cls): - missed = [] - - # check modules - - if 'modules' in cls.prerequisites: - available_modules = list(option.available['modules'].keys()) - - for module in cls.prerequisites['modules']: - if module in available_modules: - continue - - missed.append(module) - - if missed: - pytest.skip('Unit has no ' + ', '.join(missed) + ' module(s)') - - # check features - - if 'features' in cls.prerequisites: - available_features = list(option.available['features'].keys()) - - for feature in cls.prerequisites['features']: - if feature in available_features: - continue - - missed.append(feature) - - if missed: - pytest.skip(', '.join(missed) + ' feature(s) not supported') -- cgit From 16ac829c8f902d847ba63604052cc16aa4bed9ce Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 10 Dec 2020 19:28:41 +0000 Subject: Tests: added tests for PHP_AUTH_* variables. --- test/php/auth/index.php | 7 ++++++ test/test_php_application.py | 58 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 test/php/auth/index.php diff --git a/test/php/auth/index.php b/test/php/auth/index.php new file mode 100644 index 00000000..d77076d8 --- /dev/null +++ b/test/php/auth/index.php @@ -0,0 +1,7 @@ + diff --git a/test/test_php_application.py b/test/test_php_application.py index 463ec35b..ad74faa8 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -384,6 +384,64 @@ class TestPHPApplication(TestApplicationPHP): r'exec: \/\w+', body ), 'disable_functions comma exec' + def test_php_application_auth(self): + self.load('auth') + + resp = self.get() + assert resp['status'] == 200, 'status' + assert resp['headers']['X-Digest'] == 'not set', 'digest' + assert resp['headers']['X-User'] == 'not set', 'user' + assert resp['headers']['X-Password'] == 'not set', 'password' + + resp = self.get( + headers={ + 'Host': 'localhost', + 'Authorization': 'Basic dXNlcjpwYXNzd29yZA==', + 'Connection': 'close', + } + ) + assert resp['status'] == 200, 'basic status' + assert resp['headers']['X-Digest'] == 'not set', 'basic digest' + assert resp['headers']['X-User'] == 'user', 'basic user' + assert resp['headers']['X-Password'] == 'password', 'basic password' + + resp = self.get( + headers={ + 'Host': 'localhost', + 'Authorization': 'Digest username="blah", realm="", uri="/"', + 'Connection': 'close', + } + ) + assert resp['status'] == 200, 'digest status' + assert ( + resp['headers']['X-Digest'] == 'username="blah", realm="", uri="/"' + ), 'digest digest' + assert resp['headers']['X-User'] == 'not set', 'digest user' + assert resp['headers']['X-Password'] == 'not set', 'digest password' + + def test_php_application_auth_invalid(self): + self.load('auth') + + def check_auth(auth): + resp = self.get(headers={ + 'Host': 'localhost', + 'Authorization': auth, + 'Connection': 'close', + }) + + assert resp['status'] == 200, 'status' + assert resp['headers']['X-Digest'] == 'not set', 'Digest' + assert resp['headers']['X-User'] == 'not set', 'User' + assert resp['headers']['X-Password'] == 'not set', 'Password' + + check_auth('Basic dXN%cjpwYXNzd29yZA==') + check_auth('Basic XNlcjpwYXNzd29yZA==') + check_auth('Basic DdXNlcjpwYXNzd29yZA==') + check_auth('Basic blah') + check_auth('Basic') + check_auth('Digest') + check_auth('blah') + def test_php_application_disable_functions_space(self): self.load('time_exec') -- cgit From 425ffc9416391e5e41236708b655d941f560c661 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Sat, 12 Dec 2020 20:08:03 +0000 Subject: Tests: hyphen sign used unstead of underscore as more common. --- test/conftest.py | 4 ++-- test/pytest.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 5c8da65f..c14e3f68 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -31,13 +31,13 @@ def pytest_addoption(parser): help="Detailed output for tests", ) parser.addoption( - "--print_log", + "--print-log", default=False, action="store_true", help="Print unit.log to stdout in case of errors", ) parser.addoption( - "--save_log", + "--save-log", default=False, action="store_true", help="Save unit.log after the test execution", diff --git a/test/pytest.ini b/test/pytest.ini index fe86cef2..8952e8c6 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,3 @@ [pytest] -addopts = -vvv -s --print_log +addopts = -vvv -s --print-log python_functions = test_* -- cgit From 1e9def50c8ecc9f9331908b5fd46b218019a0fb0 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Mon, 14 Dec 2020 12:00:28 +0000 Subject: Isolation: fixed unmounting when mnt namespace is in place. The code had a wrong assumption that "mount namespaces" automatically unmounts process mounts when exits but this happens only with unprivileged mounts. --- src/nxt_isolation.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index 1e6323bc..cab0074b 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -676,12 +676,6 @@ nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) return; } -#if (NXT_HAVE_CLONE_NEWNS) - if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS)) { - return; - } -#endif - nxt_debug(task, "unmount all (%s)", process->name); automount = &process->isolation.automount; -- cgit From 8d65a3303bde4fa2725310cd38af311e503e75ae Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Mon, 14 Dec 2020 17:15:49 +0300 Subject: Python: WSGI environment copying moved out of request processing. The WSGI environment dictionary contains a number of static items, that are pre-initialized on application start. Then it's copied for each request to be filled with request-related data. Now this dictionary copy operation will be done between processing of requests, which should save some CPU cycles during request processing and thus reduce response latency for non-peak load periods. --- src/python/nxt_python_wsgi.c | 65 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/src/python/nxt_python_wsgi.c b/src/python/nxt_python_wsgi.c index da7b183c..77c45af5 100644 --- a/src/python/nxt_python_wsgi.c +++ b/src/python/nxt_python_wsgi.c @@ -59,6 +59,7 @@ static void nxt_python_wsgi_done(void); static void nxt_python_request_handler(nxt_unit_request_info_t *req); static PyObject *nxt_python_create_environ(nxt_python_app_conf_t *c); +static PyObject *nxt_python_copy_environ(nxt_unit_request_info_t *req); static PyObject *nxt_python_get_environ(nxt_python_ctx_t *pctx); static int nxt_python_add_sptr(nxt_python_ctx_t *pctx, PyObject *name, nxt_unit_sptr_t *sptr, uint32_t size); @@ -221,6 +222,7 @@ nxt_python_wsgi_ctx_data_alloc(void **pdata) } pctx->write = NULL; + pctx->environ = NULL; pctx->start_resp = PyCFunction_New(nxt_py_start_resp_method, (PyObject *) pctx); @@ -237,6 +239,11 @@ nxt_python_wsgi_ctx_data_alloc(void **pdata) goto fail; } + pctx->environ = nxt_python_copy_environ(NULL); + if (nxt_slow_path(pctx->environ == NULL)) { + goto fail; + } + *pdata = pctx; return NXT_UNIT_OK; @@ -258,6 +265,7 @@ nxt_python_wsgi_ctx_data_free(void *data) Py_XDECREF(pctx->start_resp); Py_XDECREF(pctx->write); + Py_XDECREF(pctx->environ); Py_XDECREF(pctx); } @@ -295,6 +303,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) int rc; PyObject *environ, *args, *response, *iterator, *item; PyObject *close, *result; + nxt_bool_t prepare_environ; nxt_python_ctx_t *pctx; pctx = req->ctx->data; @@ -305,6 +314,19 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) PyEval_RestoreThread(pctx->thread_state); + if (nxt_slow_path(pctx->environ == NULL)) { + pctx->environ = nxt_python_copy_environ(req); + + if (pctx->environ == NULL) { + prepare_environ = 0; + + rc = NXT_UNIT_ERROR; + goto done; + } + } + + prepare_environ = 1; + environ = nxt_python_get_environ(pctx); if (nxt_slow_path(environ == NULL)) { rc = NXT_UNIT_ERROR; @@ -418,6 +440,14 @@ done: pctx->req = NULL; nxt_unit_request_done(req, rc); + + if (nxt_fast_path(prepare_environ)) { + PyEval_RestoreThread(pctx->thread_state); + + pctx->environ = nxt_python_copy_environ(NULL); + + pctx->thread_state = PyEval_SaveThread(); + } } @@ -532,23 +562,30 @@ fail: static PyObject * -nxt_python_get_environ(nxt_python_ctx_t *pctx) +nxt_python_copy_environ(nxt_unit_request_info_t *req) { - int rc; - uint32_t i, j, vl; - PyObject *environ; - nxt_unit_field_t *f, *f2; - nxt_unit_request_t *r; + PyObject *environ; environ = PyDict_Copy(nxt_py_environ_ptyp); + if (nxt_slow_path(environ == NULL)) { - nxt_unit_req_error(pctx->req, + nxt_unit_req_alert(req, "Python failed to copy the \"environ\" dictionary"); - - return NULL; + nxt_python_print_exception(); } - pctx->environ = environ; + return environ; +} + + +static PyObject * +nxt_python_get_environ(nxt_python_ctx_t *pctx) +{ + int rc; + uint32_t i, j, vl; + PyObject *environ; + nxt_unit_field_t *f, *f2; + nxt_unit_request_t *r; r = pctx->req->request; @@ -628,7 +665,7 @@ nxt_python_get_environ(nxt_python_ctx_t *pctx) #undef RC - if (nxt_slow_path(PyDict_SetItem(environ, nxt_py_wsgi_input_str, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, nxt_py_wsgi_input_str, (PyObject *) pctx) != 0)) { nxt_unit_req_error(pctx->req, @@ -636,11 +673,15 @@ nxt_python_get_environ(nxt_python_ctx_t *pctx) goto fail; } + environ = pctx->environ; + pctx->environ = NULL; + return environ; fail: - Py_DECREF(environ); + Py_DECREF(pctx->environ); + pctx->environ = NULL; return NULL; } -- cgit From 66ac35e9b941500a95a069066d896793b5df3a2a Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Tue, 15 Dec 2020 11:06:49 +0000 Subject: Tests: fixed bug that disabled isolation tests. --- test/unit/check/isolation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/check/isolation.py b/test/unit/check/isolation.py index bb8feed1..fe5a41f8 100644 --- a/test/unit/check/isolation.py +++ b/test/unit/check/isolation.py @@ -135,7 +135,7 @@ def check_isolation(): body=json.dumps(conf), ) - if 'success' not in resp: + if 'success' not in resp['body']: return userns = getns('user') -- cgit From ea5844ae7b52969e2b404f36a6cb5577a3a447a2 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Tue, 15 Dec 2020 13:01:44 +0000 Subject: Added a changelog for 5e6c2b8fb3fe. --- docs/changes.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changes.xml b/docs/changes.xml index a67a671b..f8198304 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -28,6 +28,13 @@ Ruby apps with configured mutlithreading could crash on start under load. + + +Mount points were not unmounted when using "mount" namespace isolation; the +bug had appeared in 1.21.0. + + + -- cgit From d15acabfc59472457f06563a939be3c97fc6b5d9 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Dec 2020 09:55:59 +0300 Subject: Packages: introduced PYTEST_ARGS env variable for test targets. This is useful for running particular tests, e.g.: PYTEST_ARGS='test/test_respawn.py::TestRespawn::test_respawn_router' make test --- pkg/deb/Makefile | 4 ++-- pkg/rpm/Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index 8a02105c..42ee1695 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -314,7 +314,7 @@ test: unit modules test -h debuild/unit-$(VERSION)/debian/build-unit/build/$${soname} || \ ln -fs `pwd`/$${so} debuild/unit-$(VERSION)/debian/build-unit/build/$${soname} ; \ done ; \ - ( cd debuild/unit-$(VERSION)/debian/build-unit && env python3 -m pytest ) ; \ + ( cd debuild/unit-$(VERSION)/debian/build-unit && env python3 -m pytest $(PYTEST_ARGS) ) ; \ } test-debug: unit modules @@ -325,7 +325,7 @@ test-debug: unit modules test -h debuild/unit-$(VERSION)/debian/build-unit-debug/build/$${soname} || \ ln -fs `pwd`/$${so} debuild/unit-$(VERSION)/debian/build-unit-debug/build/$${soname} ; \ done ; \ - ( cd debuild/unit-$(VERSION)/debian/build-unit-debug && env python3 -m pytest ) ; \ + ( cd debuild/unit-$(VERSION)/debian/build-unit-debug && env python3 -m pytest $(PYTEST_ARGS) ) ; \ } clean: diff --git a/pkg/rpm/Makefile b/pkg/rpm/Makefile index 1944d58d..98a8ce97 100644 --- a/pkg/rpm/Makefile +++ b/pkg/rpm/Makefile @@ -274,7 +274,7 @@ test: unit modules test -h rpmbuild/BUILD/unit-$(VERSION)/build-nodebug/$${soname} || \ ln -fs `pwd`/$${so} rpmbuild/BUILD/unit-$(VERSION)/build-nodebug/$${soname} ; \ done ; \ - ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-nodebug build && env python3 -m pytest ) ; \ + ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-nodebug build && env python3 -m pytest $(PYTEST_ARGS) ) ; \ } test-debug: unit modules @@ -285,7 +285,7 @@ test-debug: unit modules test -h rpmbuild/BUILD/unit-$(VERSION)/build-debug/$${soname} || \ ln -fs `pwd`/$${so} rpmbuild/BUILD/unit-$(VERSION)/build-debug/$${soname} ; \ done ; \ - ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-debug build && env python3 -m pytest ) ; \ + ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-debug build && env python3 -m pytest $(PYTEST_ARGS) ) ; \ } clean: -- cgit From 0c1abda39110fb0dccf8853996897fe24454930d Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Dec 2020 09:56:01 +0300 Subject: Packages: the NOTICE file added to debian binaries. This resolves the following lintian error: https://lintian.debian.org/tags/missing-notice-file-for-apache-license.html While here, changed upstream changelog name to conform with the policy: https://lintian.debian.org/tags/wrong-name-for-upstream-changelog.html --- pkg/deb/debian/rules.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/deb/debian/rules.in b/pkg/deb/debian/rules.in index a4696793..3d6bfe22 100644 --- a/pkg/deb/debian/rules.in +++ b/pkg/deb/debian/rules.in @@ -104,8 +104,9 @@ install: build do.tests install -m 644 $(BUILDDIR_unit_debug)/build/libunit.a $(INSTALLDIR_dev)/usr/lib/$(DEB_HOST_MULTIARCH)/libunit-debug.a mkdir -p $(INSTALLDIR)/usr/share/doc/unit/examples install -m 644 debian/unit.example.config $(INSTALLDIR)/usr/share/doc/unit/examples/example.config - install -m 644 CHANGES $(INSTALLDIR)/usr/share/doc/unit/ + install -m 644 CHANGES $(INSTALLDIR)/usr/share/doc/unit/changelog install -m 644 README $(INSTALLDIR)/usr/share/doc/unit/ + install -m 644 NOTICE $(INSTALLDIR)/usr/share/doc/unit/ binary-indep: build install dh_testdir -- cgit From c33b10a41fc76353295bee72a53a75e96441cc49 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 16 Dec 2020 09:56:04 +0300 Subject: Packages: eliminated debuild warning about debian/rules. --- pkg/deb/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index 42ee1695..ada5f35b 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -205,6 +205,7 @@ debuild/$(SRCDIR)/debian: cat debian/rules.in | sed \ -e "s#%%CONFIGURE_ARGS%%#$(CONFIGURE_ARGS)#g" \ > debuild/$(SRCDIR)/debian/rules ; \ + chmod +x debuild/$(SRCDIR)/debian/rules ; \ } debuild/$(SRCDIR)/debian/changelog: ../../docs/changes.xml | debuild/$(SRCDIR)/debian -- cgit From 9c09bc17726ccb2ab769faec6aacbe38db1b2e0f Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Tue, 24 Nov 2020 13:25:09 +0300 Subject: Packages: dropped support for non-systemd distributions. --- pkg/deb/debian/rules.in | 2 +- pkg/deb/debian/unit.init | 74 ----------------------------- pkg/rpm/rpmbuild/SOURCES/unit.init | 88 ----------------------------------- pkg/rpm/rpmbuild/SOURCES/unit.sysconf | 1 - pkg/rpm/unit.spec.in | 58 ++++------------------- 5 files changed, 9 insertions(+), 214 deletions(-) delete mode 100644 pkg/deb/debian/unit.init delete mode 100644 pkg/rpm/rpmbuild/SOURCES/unit.init delete mode 100644 pkg/rpm/rpmbuild/SOURCES/unit.sysconf diff --git a/pkg/deb/debian/rules.in b/pkg/deb/debian/rules.in index 3d6bfe22..1e76830f 100644 --- a/pkg/deb/debian/rules.in +++ b/pkg/deb/debian/rules.in @@ -96,7 +96,7 @@ install: build do.tests dh_testroot dh_prep dh_installdirs - dh_installinit + dh_installsystemd dh_installlogrotate cd $(BUILDDIR_unit) && DESTDIR=$(INSTALLDIR) make install cd $(BUILDDIR_unit) && DESTDIR=$(INSTALLDIR_dev) make libunit-install diff --git a/pkg/deb/debian/unit.init b/pkg/deb/debian/unit.init deleted file mode 100644 index 900e97fd..00000000 --- a/pkg/deb/debian/unit.init +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -# -# unitd NGINX Unit -# -### BEGIN INIT INFO -# Provides: unitd -# Required-Start: $network $remote_fs -# Required-Stop: $network $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: NGINX Unit -# Description: NGINX Unit -### END INIT INFO -PATH=/sbin:/bin:/usr/sbin:/usr/bin -DAEMON=/usr/sbin/unitd -NAME=unit -DESC=unitd - -#includes lsb functions -. /lib/lsb/init-functions - -test -f $DAEMON || exit 0 - -umask 022 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - if start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ - --exec $DAEMON -- $DAEMON_ARGS; then - log_end_msg 0 - else - log_end_msg 1 - fi - ;; - status) - status_of_proc -p "$PIDFILE" "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - if start-stop-daemon --oknodo --stop --quiet --pidfile /var/run/$NAME.pid \ - --exec $DAEMON; then - log_end_msg 0 - else - log_end_msg 1 - fi - ;; - reload|force-reload) - echo "Not implemented." >&2 - exit 1 - ;; - restart) - log_action_begin_msg "Restarting $DESC" "$NAME" - - start-stop-daemon --stop --quiet --pidfile \ - /var/run/$NAME.pid --exec $DAEMON || true - sleep 1 - if start-stop-daemon --start --quiet --pidfile \ - /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_ARGS; then - log_end_msg 0 - else - log_end_msg 1 - fi - ;; - *) - echo "Usage: /etc/init.d/$NAME {start|status|stop|restart|reload|force-reload}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.init b/pkg/rpm/rpmbuild/SOURCES/unit.init deleted file mode 100644 index e1aacd81..00000000 --- a/pkg/rpm/rpmbuild/SOURCES/unit.init +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/sh -# -# unitd NGINX Unit -# -# chkconfig: - 86 14 -# description: NGINX Unit - -### BEGIN INIT INFO -# Provides: unitd -# Required-Start: $local_fs $network $named $syslog -# Required-Stop: $local_fs $network $named $syslog -# Default-Start: -# Default-Stop: 0 1 2 3 4 5 6 -# Short-Description: NGINX Unit -# Description: NGINX Unit -### END INIT INFO - -# Source function library. -. /etc/rc.d/init.d/functions - -exec="/usr/sbin/unitd" -prog="unitd" - -[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog - -lockfile=/var/lock/subsys/$prog - -start() { - [ -x $exec ] || exit 5 - echo -n $"Starting $prog: " - daemon $exec $UNITD_OPTIONS - retval=$? - echo - [ $retval -eq 0 ] && touch $lockfile - return $retval -} - -stop() { - echo -n $"Stopping $prog: " - killproc $prog - retval=$? - echo - [ $retval -eq 0 ] && rm -f $lockfile - return $retval -} - -restart() { - stop - start -} - -rh_status() { - status $prog -} - -rh_status_q() { - rh_status &>/dev/null -} - - -case "$1" in - start) - rh_status_q && exit 0 - $1 - ;; - stop) - rh_status_q || exit 0 - $1 - ;; - restart) - $1 - ;; - reload|force-reload) - echo "Not implemented." >&2 - exit 1 - ;; - status) - rh_status - ;; - condrestart|try-restart) - rh_status_q || exit 0 - restart - ;; - *) - echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart}" - exit 2 -esac -exit $? diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.sysconf b/pkg/rpm/rpmbuild/SOURCES/unit.sysconf deleted file mode 100644 index 9146bdac..00000000 --- a/pkg/rpm/rpmbuild/SOURCES/unit.sysconf +++ /dev/null @@ -1 +0,0 @@ -UNITD_OPTIONS="--log /var/log/unit/unit.log --pid /var/run/unit/unit.pid" diff --git a/pkg/rpm/unit.spec.in b/pkg/rpm/unit.spec.in index 7fc950ec..f2a77b61 100644 --- a/pkg/rpm/unit.spec.in +++ b/pkg/rpm/unit.spec.in @@ -1,19 +1,7 @@ # distribution specific definitions -%define use_systemd (0%{?rhel} >= 7 || 0%{?fedora} >= 19 || 0%{?suse_version} >= 1315) %define bdir %{_builddir}/%{name}-%{version} %define dotests 0 -%if ( 0%{?rhel} == 5 || 0%{?rhel} == 6 ) -Requires: initscripts >= 8.36 -%endif - -%if %{use_systemd} -BuildRequires: systemd -Requires(post): systemd -Requires(preun): systemd -Requires(postun): systemd -%endif - %if 0%{?rhel}%{?fedora} BuildRequires: gcc BuildRequires: openssl-devel @@ -48,12 +36,14 @@ Group: System Environment/Daemons Source0: unit-%{version}.tar.gz Source1: unit.service -Source2: unit.init -Source3: unit.sysconf -Source4: unit.example.config -Source5: unit.logrotate +Source2: unit.example.config +Source3: unit.logrotate BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRequires: systemd +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd %description NGINX Unit is a runtime and delivery environment for modern distributed @@ -116,17 +106,12 @@ DESTDIR=%{buildroot} make unitd-install libunit-install %{__mkdir} -p %{buildroot}%{_sharedstatedir}/unit %{__mkdir} -p %{buildroot}%{_localstatedir}/log/unit %{__mkdir} -p %{buildroot}%{_localstatedir}/run/unit -%if ! %{use_systemd} -%{__mkdir} -p %{buildroot}%{_sysconfdir}/sysconfig -%{__install} -m 644 -p %{SOURCE3} \ - %{buildroot}%{_sysconfdir}/sysconfig/unitd -%endif %{__mkdir} -p %{buildroot}%{_sysconfdir}/logrotate.d -%{__install} -m 644 -p %{SOURCE5} \ +%{__install} -m 644 -p %{SOURCE3} \ %{buildroot}%{_sysconfdir}/logrotate.d/unit %{__mkdir} -p %{buildroot}%{_sysconfdir}/unit %{__mkdir} -p %{buildroot}%{_datadir}/doc/unit/examples -%{__install} -m 644 -p %{SOURCE4} \ +%{__install} -m 644 -p %{SOURCE2} \ %{buildroot}%{_datadir}/doc/unit/examples/example.config %{__install} -m 644 -p CHANGES \ %{buildroot}%{_datadir}/doc/unit/ @@ -135,14 +120,8 @@ DESTDIR=%{buildroot} make unitd-install libunit-install %{__install} -m 644 -p README \ %{buildroot}%{_datadir}/doc/unit/ -# init scripts -%if %{use_systemd} %{__rm} -rf %{buildroot}%{_initrddir}/ %{__install} -p -D -m 0644 %{SOURCE1} %{buildroot}%{_unitdir}/unit.service -%else -%{__mkdir} -p %{buildroot}%{_initrddir} -%{__install} -p -D -m 0755 %{SOURCE2} %{buildroot}%{_initrddir}/unit -%endif QA_SKIP_BUILD_ROOT=1 export QA_SKIP_BUILD_ROOT @@ -164,11 +143,7 @@ cat /dev/null > debugsourcefiles.list %post if [ $1 -eq 1 ]; then -%if %{use_systemd} /usr/bin/systemctl preset unit.service >/dev/null 2>&1 ||: -%else - /sbin/chkconfig --add unit -%endif cat </dev/null 2>&1 ||: /usr/bin/systemctl stop unit.service >/dev/null 2>&1 ||: -%else - /sbin/service unit stop >/dev/null 2>&1 - /sbin/chkconfig --del unit -%endif fi %postun -%if %{use_systemd} /usr/bin/systemctl daemon-reload >/dev/null 2>&1 ||: -%endif if [ $1 -ge 1 ]; then -%if %{use_systemd} /usr/bin/systemctl try-restart unit.service >/dev/null 2>&1 ||: -%else - /sbin/service unit condrestart >/dev/null 2>&1 ||: -%endif fi %files @@ -211,14 +175,8 @@ fi %attr(0755,root,root) %{_sbindir}/unitd %attr(0755,root,root) %{_sbindir}/unitd-debug %dir %{_sysconfdir}/unit -%if %{use_systemd} %{_unitdir}/unit.service %dir %attr(0755,root,root) %ghost %{_localstatedir}/run/unit -%else -%config(noreplace) %{_sysconfdir}/sysconfig/unitd -%dir %attr(0755,root,root) %{_localstatedir}/run/unit -%{_initrddir}/unit -%endif %dir %{_datadir}/doc/unit %{_datadir}/doc/unit/* %dir %{_libdir}/unit/modules -- cgit From a5fa9673d98c9c814c01ddfce8bcfbec6fcc02e8 Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Tue, 24 Nov 2020 14:27:06 +0300 Subject: Packages: added pcre2 to build depends. While at it, propagate unit build depends to modules. --- pkg/deb/debian.module/control.in | 4 +++- pkg/deb/debian/control | 3 ++- pkg/rpm/unit.module.spec.in | 11 +++++++++++ pkg/rpm/unit.spec.in | 2 ++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/deb/debian.module/control.in b/pkg/deb/debian.module/control.in index e9b8b8e2..7e28f5e9 100644 --- a/pkg/deb/debian.module/control.in +++ b/pkg/deb/debian.module/control.in @@ -3,7 +3,9 @@ Section: admin Priority: extra Maintainer: Andrei Belov Build-Depends: debhelper (>= 9), - linux-libc-dev%%MODULE_BUILD_DEPENDS%% + linux-libc-dev, + libssl-dev, + libpcre2-dev%%MODULE_BUILD_DEPENDS%% Standards-Version: 3.9.5 Homepage: https://unit.nginx.org diff --git a/pkg/deb/debian/control b/pkg/deb/debian/control index 9828b6ab..a8e8cdc4 100644 --- a/pkg/deb/debian/control +++ b/pkg/deb/debian/control @@ -4,7 +4,8 @@ Priority: extra Maintainer: Andrei Belov Build-Depends: debhelper (>= 9), linux-libc-dev, - libssl-dev + libssl-dev, + libpcre2-dev Standards-Version: 3.9.5 Homepage: https://unit.nginx.org diff --git a/pkg/rpm/unit.module.spec.in b/pkg/rpm/unit.module.spec.in index 2ef4ff1b..39083e66 100644 --- a/pkg/rpm/unit.module.spec.in +++ b/pkg/rpm/unit.module.spec.in @@ -7,6 +7,15 @@ %define dist .el7 %endif +%if 0%{?rhel}%{?fedora} +BuildRequires: gcc +BuildRequires: openssl-devel +%endif + +%if 0%{?suse_version} >= 1315 +BuildRequires: libopenssl-devel +%endif + %define unit_version %%UNIT_VERSION%% %define unit_release %%UNIT_RELEASE%%%{?dist}.ngx @@ -29,6 +38,8 @@ Source0: unit-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRequires: pcre2-devel + Requires: unit == %%UNIT_VERSION%%-%%UNIT_RELEASE%%%{?dist}.ngx %description diff --git a/pkg/rpm/unit.spec.in b/pkg/rpm/unit.spec.in index f2a77b61..506d1964 100644 --- a/pkg/rpm/unit.spec.in +++ b/pkg/rpm/unit.spec.in @@ -45,6 +45,8 @@ Requires(post): systemd Requires(preun): systemd Requires(postun): systemd +BuildRequires: pcre2-devel + %description NGINX Unit is a runtime and delivery environment for modern distributed applications. It runs the application code in multiple languages -- cgit From 53d847615b270daf706373d65ac5f5d2101f36d9 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Thu, 17 Dec 2020 14:30:18 +0300 Subject: Packages: run non-privileged processes under "unit" user. --- pkg/deb/Makefile | 2 ++ pkg/deb/debian.module/unit.example-go-config | 1 - pkg/deb/debian.module/unit.example-perl-config | 1 - pkg/deb/debian.module/unit.example-php-config | 1 - pkg/deb/debian.module/unit.example-python-config | 1 - .../debian.module/unit.example-python2.7-config | 1 - .../debian.module/unit.example-python3.4-config | 1 - .../debian.module/unit.example-python3.5-config | 1 - .../debian.module/unit.example-python3.6-config | 1 - .../debian.module/unit.example-python3.7-config | 1 - .../debian.module/unit.example-python3.8-config | 1 - pkg/deb/debian.module/unit.example-ruby-config | 1 - pkg/deb/debian/unit.example.config | 4 --- pkg/deb/debian/unit.postinst | 34 ++++++++++++++++++++++ pkg/deb/debian/unit.preinst | 1 + pkg/rpm/Makefile | 2 ++ pkg/rpm/rpmbuild/SOURCES/unit.example-go-config | 1 - pkg/rpm/rpmbuild/SOURCES/unit.example-perl-config | 1 - pkg/rpm/rpmbuild/SOURCES/unit.example-php-config | 1 - .../rpmbuild/SOURCES/unit.example-python-config | 1 - .../rpmbuild/SOURCES/unit.example-python27-config | 1 - .../rpmbuild/SOURCES/unit.example-python34-config | 1 - .../rpmbuild/SOURCES/unit.example-python35-config | 1 - .../rpmbuild/SOURCES/unit.example-python36-config | 1 - .../rpmbuild/SOURCES/unit.example-python37-config | 1 - .../rpmbuild/SOURCES/unit.example-python38-config | 1 - pkg/rpm/rpmbuild/SOURCES/unit.example-ruby-config | 1 - pkg/rpm/rpmbuild/SOURCES/unit.example.config | 4 --- pkg/rpm/unit.spec.in | 19 ++++++++++++ 29 files changed, 58 insertions(+), 30 deletions(-) diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index ada5f35b..6ce7373b 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -157,6 +157,8 @@ CONFIGURE_ARGS=\ --pid=/var/run/unit.pid \ --log=/var/log/unit.log \ --tmp=/var/tmp \ + --user=unit \ + --group=unit \ --tests \ --openssl diff --git a/pkg/deb/debian.module/unit.example-go-config b/pkg/deb/debian.module/unit.example-go-config index a2c91e80..8aa65939 100644 --- a/pkg/deb/debian.module/unit.example-go-config +++ b/pkg/deb/debian.module/unit.example-go-config @@ -2,7 +2,6 @@ "applications": { "example_go": { "type": "external", - "user": "nobody", "executable": "/tmp/go-app" } }, diff --git a/pkg/deb/debian.module/unit.example-perl-config b/pkg/deb/debian.module/unit.example-perl-config index 031928ce..2182fc46 100644 --- a/pkg/deb/debian.module/unit.example-perl-config +++ b/pkg/deb/debian.module/unit.example-perl-config @@ -2,7 +2,6 @@ "applications": { "example_perl": { "type": "perl", - "user": "nobody", "processes": 1, "working_directory": "/usr/share/doc/unit-perl/examples/perl-app", "script": "/usr/share/doc/unit-perl/examples/perl-app/index.pl" diff --git a/pkg/deb/debian.module/unit.example-php-config b/pkg/deb/debian.module/unit.example-php-config index 8f23c984..9673385f 100644 --- a/pkg/deb/debian.module/unit.example-php-config +++ b/pkg/deb/debian.module/unit.example-php-config @@ -2,7 +2,6 @@ "applications": { "example_php": { "type": "php", - "user": "nobody", "processes": 2, "root": "/usr/share/doc/unit-php/examples/phpinfo-app", "index": "index.php" diff --git a/pkg/deb/debian.module/unit.example-python-config b/pkg/deb/debian.module/unit.example-python-config index d612c89d..b3d3a2e5 100644 --- a/pkg/deb/debian.module/unit.example-python-config +++ b/pkg/deb/debian.module/unit.example-python-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-python2.7-config b/pkg/deb/debian.module/unit.example-python2.7-config index bede8899..4f1d16c9 100644 --- a/pkg/deb/debian.module/unit.example-python2.7-config +++ b/pkg/deb/debian.module/unit.example-python2.7-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 2.7", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python2.7/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-python3.4-config b/pkg/deb/debian.module/unit.example-python3.4-config index dd496bd8..e6d90acf 100644 --- a/pkg/deb/debian.module/unit.example-python3.4-config +++ b/pkg/deb/debian.module/unit.example-python3.4-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.4", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python3.4/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-python3.5-config b/pkg/deb/debian.module/unit.example-python3.5-config index 2be6de4a..480327ec 100644 --- a/pkg/deb/debian.module/unit.example-python3.5-config +++ b/pkg/deb/debian.module/unit.example-python3.5-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.5", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python3.5/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-python3.6-config b/pkg/deb/debian.module/unit.example-python3.6-config index a77e8e07..543024ff 100644 --- a/pkg/deb/debian.module/unit.example-python3.6-config +++ b/pkg/deb/debian.module/unit.example-python3.6-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.6", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python3.6/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-python3.7-config b/pkg/deb/debian.module/unit.example-python3.7-config index 9b13c058..e7b8dbc3 100644 --- a/pkg/deb/debian.module/unit.example-python3.7-config +++ b/pkg/deb/debian.module/unit.example-python3.7-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.7", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python3.7/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-python3.8-config b/pkg/deb/debian.module/unit.example-python3.8-config index 435e025f..dc649e30 100644 --- a/pkg/deb/debian.module/unit.example-python3.8-config +++ b/pkg/deb/debian.module/unit.example-python3.8-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.8", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python3.8/examples/python-app", "module": "wsgi" diff --git a/pkg/deb/debian.module/unit.example-ruby-config b/pkg/deb/debian.module/unit.example-ruby-config index 15a92735..930aa987 100644 --- a/pkg/deb/debian.module/unit.example-ruby-config +++ b/pkg/deb/debian.module/unit.example-ruby-config @@ -2,7 +2,6 @@ "applications": { "example_ruby": { "type": "ruby", - "user": "nobody", "processes": 2, "script": "/usr/share/doc/unit-ruby/examples/ruby-app.ru" } diff --git a/pkg/deb/debian/unit.example.config b/pkg/deb/debian/unit.example.config index 5610cb3a..66695327 100644 --- a/pkg/deb/debian/unit.example.config +++ b/pkg/deb/debian/unit.example.config @@ -2,7 +2,6 @@ "applications": { "example_php": { "type": "php", - "user": "nobody", "processes": 2, "root": "/usr/share/doc/unit/examples/php-app", "index": "index.php" @@ -10,7 +9,6 @@ "example_python": { "type": "python", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit/examples/python-app", "module": "wsgi" @@ -18,13 +16,11 @@ "example_go": { "type": "external", - "user": "nobody", "executable": "/tmp/go-app" }, "example_perl": { "type": "perl", - "user": "nobody", "processes": 1, "working_directory": "/usr/share/doc/unit-perl/examples/perl-app", "script": "/usr/share/doc/unit-perl/examples/perl-app/index.pl" diff --git a/pkg/deb/debian/unit.postinst b/pkg/deb/debian/unit.postinst index 76375a2b..8aa476b3 100755 --- a/pkg/deb/debian/unit.postinst +++ b/pkg/deb/debian/unit.postinst @@ -6,6 +6,40 @@ if [ "$1" != "configure" ]; then exit 0 fi +if [ -n "$2" ]; then + if dpkg --compare-versions "${2%%-*}" le "1.21.0"; then + cat </dev/null; then + addgroup --system unit >/dev/null +fi + +if ! getent passwd unit >/dev/null; then + adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit >/dev/null +fi + #DEBHELPER# exit 0 diff --git a/pkg/deb/debian/unit.preinst b/pkg/deb/debian/unit.preinst index d4be468d..bd513788 100644 --- a/pkg/deb/debian/unit.preinst +++ b/pkg/deb/debian/unit.preinst @@ -17,6 +17,7 @@ Online documentation is available at https://unit.nginx.org/ ---------------------------------------------------------------------- BANNER ;; + upgrade) ;; diff --git a/pkg/rpm/Makefile b/pkg/rpm/Makefile index 98a8ce97..1fefb262 100644 --- a/pkg/rpm/Makefile +++ b/pkg/rpm/Makefile @@ -149,6 +149,8 @@ CONFIGURE_ARGS=\ --pid=/var/run/unit/unit.pid \ --log=/var/log/unit/unit.log \ --tmp=/var/tmp \ + --user=unit \ + --group=unit \ --tests \ --openssl diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-go-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-go-config index a2c91e80..8aa65939 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-go-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-go-config @@ -2,7 +2,6 @@ "applications": { "example_go": { "type": "external", - "user": "nobody", "executable": "/tmp/go-app" } }, diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-perl-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-perl-config index 031928ce..2182fc46 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-perl-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-perl-config @@ -2,7 +2,6 @@ "applications": { "example_perl": { "type": "perl", - "user": "nobody", "processes": 1, "working_directory": "/usr/share/doc/unit-perl/examples/perl-app", "script": "/usr/share/doc/unit-perl/examples/perl-app/index.pl" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-php-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-php-config index 8f23c984..9673385f 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-php-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-php-config @@ -2,7 +2,6 @@ "applications": { "example_php": { "type": "php", - "user": "nobody", "processes": 2, "root": "/usr/share/doc/unit-php/examples/phpinfo-app", "index": "index.php" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python-config index d612c89d..b3d3a2e5 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python27-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python27-config index 7541fcb3..094e6621 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python27-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python27-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 2.7", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python27/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python34-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python34-config index b64e570c..15063c5e 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python34-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python34-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.4", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python34/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python35-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python35-config index 025f3428..f9923a49 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python35-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python35-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.5", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python35/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python36-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python36-config index 825cabc4..ef31c781 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python36-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python36-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.6", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python36/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python37-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python37-config index 7f5e52f1..904af440 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python37-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python37-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.7", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python37/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-python38-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-python38-config index 25003869..c98d1a52 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-python38-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-python38-config @@ -2,7 +2,6 @@ "applications": { "example_python": { "type": "python 3.8", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python38/examples/python-app", "module": "wsgi" diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example-ruby-config b/pkg/rpm/rpmbuild/SOURCES/unit.example-ruby-config index 15a92735..930aa987 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example-ruby-config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example-ruby-config @@ -2,7 +2,6 @@ "applications": { "example_ruby": { "type": "ruby", - "user": "nobody", "processes": 2, "script": "/usr/share/doc/unit-ruby/examples/ruby-app.ru" } diff --git a/pkg/rpm/rpmbuild/SOURCES/unit.example.config b/pkg/rpm/rpmbuild/SOURCES/unit.example.config index 6fe35e2f..4855a954 100644 --- a/pkg/rpm/rpmbuild/SOURCES/unit.example.config +++ b/pkg/rpm/rpmbuild/SOURCES/unit.example.config @@ -2,7 +2,6 @@ "applications": { "example_php": { "type": "php", - "user": "nobody", "processes": 2, "root": "/usr/share/doc/unit-php/examples/phpinfo-app", "index": "index.php" @@ -10,7 +9,6 @@ "example_python": { "type": "python", - "user": "nobody", "processes": 2, "path": "/usr/share/doc/unit-python/examples/python-app", "module": "wsgi" @@ -18,13 +16,11 @@ "example_go": { "type": "external", - "user": "nobody", "executable": "/tmp/go-app" }, "example_perl": { "type": "perl", - "user": "nobody", "processes": 1, "working_directory": "/usr/share/doc/unit-perl/examples/perl-app", "script": "/usr/share/doc/unit-perl/examples/perl-app/index.pl" diff --git a/pkg/rpm/unit.spec.in b/pkg/rpm/unit.spec.in index 506d1964..15853cf1 100644 --- a/pkg/rpm/unit.spec.in +++ b/pkg/rpm/unit.spec.in @@ -145,6 +145,10 @@ cat /dev/null > debugsourcefiles.list %post if [ $1 -eq 1 ]; then + getent group unit >/dev/null || groupadd -r unit + getent passwd unit >/dev/null || \ + useradd -r -g unit -s /sbin/nologin \ + -d /nonexistent -c "unit user" unit /usr/bin/systemctl preset unit.service >/dev/null 2>&1 ||: cat </dev/null 2>&1 ||: fi +%triggerpostun -- unit < 1.22.0 +cat < Date: Thu, 17 Dec 2020 19:27:44 +0300 Subject: Router: fixed crash in OOSM processing. Multithreaded application may create different shared memory segments in different threads. The segments then passed to different router threads. Because of this multithreading, the order of adding incoming segments is not determined and there can be situation when some of the incoming segments are not initialized yet. This patch simply adds check for NULL to skip non-initialized segments. Crash reproduced during load tests with high number of simultaneous connections (1024 and more). --- docs/changes.xml | 6 ++++++ src/nxt_router.c | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index f8198304..ca4ffe5f 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -35,6 +35,12 @@ bug had appeared in 1.21.0. + + +the router process could crash with multithreaded applications under high load. + + + diff --git a/src/nxt_router.c b/src/nxt_router.c index 871602e4..0416dea0 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -5373,7 +5373,7 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_bool_t ack; nxt_process_t *process; nxt_free_map_t *m; - nxt_port_mmap_header_t *hdr; + nxt_port_mmap_handler_t *mmap_handler; nxt_debug(task, "oosm in %PI", msg->port_msg.pid); @@ -5394,8 +5394,13 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_thread_mutex_lock(&process->incoming.mutex); for (i = 0; i < process->incoming.size; i++) { - hdr = process->incoming.elts[i].mmap_handler->hdr; - m = hdr->free_map; + mmap_handler = process->incoming.elts[i].mmap_handler; + + if (nxt_slow_path(mmap_handler == NULL)) { + continue; + } + + m = mmap_handler->hdr->free_map; for (mi = 0; mi < MAX_FREE_IDX; mi++) { if (m[mi] != 0) { -- cgit From 7389a50835696fe256c5decf31bec129f1d59bbf Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Fri, 18 Dec 2020 00:25:27 +0300 Subject: Limiting app queue notifications count in socket. Under high load, a queue synchonization issue may occur, starting from the steady state when an app queue message is dequeued immediately after it has been enqueued. In this state, the router always puts the first message in the queue and is forced to notify the app about a new message in an empty queue using a socket pair. On the other hand, the application dequeues and processes the message without reading the notification from the socket, so the socket buffer overflows with notifications. The issue was reproduced during Unit load tests. After a socket buffer overflow, the router is unable to notify the app about a new first message. When another message is enqueued, a notification is not required, so the queue grows without being read by the app. As a result, request processing stops. This patch changes the notification algorithm by counting the notifications in the pipe instead of getting the number of messages in the queue. --- docs/changes.xml | 7 +++++++ src/nxt_app_queue.h | 18 ++++++++++++------ src/nxt_unit.c | 7 ++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index ca4ffe5f..f157e9bf 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -41,6 +41,13 @@ the router process could crash with multithreaded applications under high load. + + +applications could stop processing new requests under high load; the bug had +appeared in 1.19.0. + + + diff --git a/src/nxt_app_queue.h b/src/nxt_app_queue.h index 127cb8f3..a1cc2f11 100644 --- a/src/nxt_app_queue.h +++ b/src/nxt_app_queue.h @@ -23,7 +23,7 @@ typedef struct { typedef struct { - nxt_app_nncq_atomic_t nitems; + nxt_app_nncq_atomic_t notified; nxt_app_nncq_t free_items; nxt_app_nncq_t queue; nxt_app_queue_item_t items[NXT_APP_QUEUE_SIZE]; @@ -42,7 +42,7 @@ nxt_app_queue_init(nxt_app_queue_t volatile *q) nxt_app_nncq_enqueue(&q->free_items, i); } - q->nitems = 0; + q->notified = 0; } @@ -50,6 +50,7 @@ nxt_inline nxt_int_t nxt_app_queue_send(nxt_app_queue_t volatile *q, const void *p, uint8_t size, uint32_t tracking, int *notify, uint32_t *cookie) { + int n; nxt_app_queue_item_t *qi; nxt_app_nncq_atomic_t i; @@ -67,16 +68,23 @@ nxt_app_queue_send(nxt_app_queue_t volatile *q, const void *p, nxt_app_nncq_enqueue(&q->queue, i); - i = nxt_atomic_fetch_add(&q->nitems, 1); + n = nxt_atomic_cmp_set(&q->notified, 0, 1); if (notify != NULL) { - *notify = (i == 0); + *notify = n; } return NXT_OK; } +nxt_inline void +nxt_app_queue_notification_received(nxt_app_queue_t volatile *q) +{ + q->notified = 0; +} + + nxt_inline nxt_bool_t nxt_app_queue_cancel(nxt_app_queue_t volatile *q, uint32_t cookie, uint32_t tracking) @@ -110,8 +118,6 @@ nxt_app_queue_recv(nxt_app_queue_t volatile *q, void *p, uint32_t *cookie) nxt_app_nncq_enqueue(&q->free_items, i); - nxt_atomic_fetch_add(&q->nitems, -1); - return res; } diff --git a/src/nxt_unit.c b/src/nxt_unit.c index b6904ce9..2cdc75f8 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -6092,7 +6092,10 @@ static int nxt_unit_shared_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, nxt_unit_read_buf_t *rbuf) { - int res; + int res; + nxt_unit_port_impl_t *port_impl; + + port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); retry: @@ -6105,6 +6108,8 @@ retry: } if (nxt_unit_is_read_queue(rbuf)) { + nxt_app_queue_notification_received(port_impl->queue); + nxt_unit_debug(ctx, "port{%d,%d} recv %d read_queue", (int) port->id.pid, (int) port->id.id, (int) rbuf->size); -- cgit From 7b669ed866896afbf26ab6bc0737fe7c8f9c2ec5 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Fri, 18 Dec 2020 00:25:28 +0300 Subject: Libunit: fixed shared memory waiting. The nxt_unit_ctx_port_recv() function may return the NXT_UNIT_AGAIN code, in which case an attempt to reread the message should be made. The issue was reproduced in load testing with response sizes 16k and up. In the rare case of a NXT_UNIT_AGAIN result, a buffer of size -1 was processed, which triggered a 'message too small' alert; after that, the app process was terminated. --- docs/changes.xml | 7 +++++++ src/nxt_unit.c | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changes.xml b/docs/changes.xml index f157e9bf..6bd30cab 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -48,6 +48,13 @@ appeared in 1.19.0. + + +application processes could terminate unexpectedly under high load; the bug +had appeared in 1.19.0. + + + diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 2cdc75f8..39e7f076 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -3606,7 +3606,10 @@ nxt_unit_wait_shm_ack(nxt_unit_ctx_t *ctx) return NXT_UNIT_ERROR; } - res = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); + do { + res = nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); + } while (res == NXT_UNIT_AGAIN); + if (res == NXT_UNIT_ERROR) { nxt_unit_read_buf_release(ctx, rbuf); -- cgit From 03436d2ec2ab485b4f3196690e9a267bf0d42d30 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Mon, 21 Dec 2020 11:00:46 +0000 Subject: Tests: introduced a separate cache directory for Go builds. The Go compiler can't detect changes to C header files when compiling CGO applications, and then this leads to Go test samples being linked with wrong libunit. This patch creates a new cache directory reused throughout the test suite. --- test/conftest.py | 4 ++++ test/unit/applications/lang/go.py | 1 + 2 files changed, 5 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index c14e3f68..c2a320de 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -69,6 +69,9 @@ def pytest_configure(config): option.architecture = platform.architecture()[0] option.system = platform.system() + option.cache_dir = tempfile.mkdtemp(prefix='unit-test-cache-') + public_dir(option.cache_dir) + # set stdout to non-blocking if option.detailed or option.print_log: @@ -434,3 +437,4 @@ def is_su(request): def pytest_sessionfinish(session): unit_stop() + shutil.rmtree(option.cache_dir) diff --git a/test/unit/applications/lang/go.py b/test/unit/applications/lang/go.py index 70f9d58c..a17b1af4 100644 --- a/test/unit/applications/lang/go.py +++ b/test/unit/applications/lang/go.py @@ -12,6 +12,7 @@ class TestApplicationGo(TestApplicationProto): env = os.environ.copy() env['GOPATH'] = option.current_dir + '/build/go' + env['GOCACHE'] = option.cache_dir + '/go' if static: args = [ -- cgit From 65295c81413cff75eddef2845f09159711e553df Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Tue, 22 Dec 2020 12:57:24 +0300 Subject: Packages: check and create unit user on each post-script invocation. --- pkg/rpm/unit.spec.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/rpm/unit.spec.in b/pkg/rpm/unit.spec.in index 15853cf1..08db73e2 100644 --- a/pkg/rpm/unit.spec.in +++ b/pkg/rpm/unit.spec.in @@ -144,11 +144,11 @@ cat /dev/null > debugsourcefiles.list %{__rm} -rf %{buildroot} %post +getent group unit >/dev/null || groupadd -r unit +getent passwd unit >/dev/null || \ + useradd -r -g unit -s /sbin/nologin \ + -d /nonexistent -c "unit user" unit if [ $1 -eq 1 ]; then - getent group unit >/dev/null || groupadd -r unit - getent passwd unit >/dev/null || \ - useradd -r -g unit -s /sbin/nologin \ - -d /nonexistent -c "unit user" unit /usr/bin/systemctl preset unit.service >/dev/null 2>&1 ||: cat < Date: Tue, 22 Dec 2020 17:53:41 +0300 Subject: Python: multiple values in the "path" option. --- docs/changes.xml | 6 +++ src/nxt_application.h | 14 +++---- src/nxt_conf_validation.c | 35 +++++++++++++++- src/nxt_main_process.c | 2 +- src/python/nxt_python.c | 103 ++++++++++++++++++++++++++++++++++------------ 5 files changed, 124 insertions(+), 36 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index 6bd30cab..db7a2799 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -9,6 +9,12 @@ date="" time="" packager="Andrei Belov <defan@nginx.com>"> + + +ability to specify multiple directories in the "path" option of Python apps. + + + invalid HTTP responses were generated for some unusual status codes. diff --git a/src/nxt_application.h b/src/nxt_application.h index 5632f56f..632c5632 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -47,13 +47,13 @@ typedef struct { typedef struct { - char *home; - nxt_str_t path; - nxt_str_t module; - char *callable; - nxt_str_t protocol; - uint32_t threads; - uint32_t thread_stack_size; + char *home; + nxt_conf_value_t *path; + nxt_str_t module; + char *callable; + nxt_str_t protocol; + uint32_t threads; + uint32_t thread_stack_size; } nxt_python_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index acb2e3de..67fa3095 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -96,6 +96,10 @@ static nxt_int_t nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_python_path(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_python_path_element(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value); static nxt_int_t nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, @@ -491,7 +495,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { .type = NXT_CONF_VLDT_STRING, }, { .name = nxt_string("path"), - .type = NXT_CONF_VLDT_STRING, + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_python_path, }, { .name = nxt_string("module"), .type = NXT_CONF_VLDT_STRING, @@ -1376,6 +1381,34 @@ nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } +static nxt_int_t +nxt_conf_vldt_python_path(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + if (nxt_conf_type(value) == NXT_CONF_ARRAY) { + return nxt_conf_vldt_array_iterator(vldt, value, + &nxt_conf_vldt_python_path_element); + } + + /* NXT_CONF_STRING */ + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_python_path_element(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value) +{ + if (nxt_conf_type(value) != NXT_CONF_STRING) { + return nxt_conf_vldt_error(vldt, "The \"path\" array must contain " + "only string values."); + } + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 0cde435b..2916f0ab 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -182,7 +182,7 @@ static nxt_conf_map_t nxt_python_app_conf[] = { { nxt_string("path"), - NXT_CONF_MAP_STR, + NXT_CONF_MAP_PTR, offsetof(nxt_common_app_conf_t, u.python.path), }, diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c index faf0c0e1..d8204937 100644 --- a/src/python/nxt_python.c +++ b/src/python/nxt_python.c @@ -24,6 +24,7 @@ typedef struct { static nxt_int_t nxt_python_start(nxt_task_t *task, nxt_process_data_t *data); +static nxt_int_t nxt_python_set_path(nxt_task_t *task, nxt_conf_value_t *value); static int nxt_python_init_threads(nxt_python_app_conf_t *c); static int nxt_python_ready_handler(nxt_unit_ctx_t *ctx); static void *nxt_python_thread_func(void *main_ctx); @@ -67,7 +68,7 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) int rc; char *nxt_py_module; size_t len; - PyObject *obj, *pypath, *module; + PyObject *obj, *module; nxt_str_t proto; const char *callable; nxt_unit_ctx_t *unit_ctx; @@ -162,38 +163,18 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) } nxt_py_stderr_flush = PyObject_GetAttrString(obj, "flush"); + + /* obj is a Borrowed reference. */ + obj = NULL; + if (nxt_slow_path(nxt_py_stderr_flush == NULL)) { nxt_alert(task, "Python failed to get \"flush\" attribute of " "\"sys.stderr\" object"); goto fail; } - /* obj is a Borrowed reference. */ - - if (c->path.length > 0) { - obj = PyString_FromStringAndSize((char *) c->path.start, - c->path.length); - - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to create string object \"%V\"", - &c->path); - goto fail; - } - - pypath = PySys_GetObject((char *) "path"); - - if (nxt_slow_path(pypath == NULL)) { - nxt_alert(task, "Python failed to get \"sys.path\" list"); - goto fail; - } - - if (nxt_slow_path(PyList_Insert(pypath, 0, obj) != 0)) { - nxt_alert(task, "Python failed to insert \"%V\" into \"sys.path\"", - &c->path); - goto fail; - } - - Py_DECREF(obj); + if (nxt_slow_path(nxt_python_set_path(task, c->path) != NXT_OK)) { + goto fail; } obj = Py_BuildValue("[s]", "unit"); @@ -317,6 +298,74 @@ fail: } +static nxt_int_t +nxt_python_set_path(nxt_task_t *task, nxt_conf_value_t *value) +{ + int ret; + PyObject *path, *sys; + nxt_str_t str; + nxt_uint_t n; + nxt_conf_value_t *array; + + if (value == NULL) { + return NXT_OK; + } + + sys = PySys_GetObject((char *) "path"); + if (nxt_slow_path(sys == NULL)) { + nxt_alert(task, "Python failed to get \"sys.path\" list"); + return NXT_ERROR; + } + + /* sys is a Borrowed reference. */ + + if (nxt_conf_type(value) == NXT_CONF_STRING) { + n = 0; + goto value_is_string; + } + + /* NXT_CONF_ARRAY */ + array = value; + + n = nxt_conf_array_elements_count(array); + + while (n != 0) { + n--; + + /* + * Insertion in front of existing paths starting from the last element + * to preserve original order while giving priority to the values + * specified in the "path" option. + */ + + value = nxt_conf_get_array_element(array, n); + + value_is_string: + + nxt_conf_get_string(value, &str); + + path = PyString_FromStringAndSize((char *) str.start, str.length); + if (nxt_slow_path(path == NULL)) { + nxt_alert(task, "Python failed to create string object \"%V\"", + &str); + return NXT_ERROR; + } + + ret = PyList_Insert(sys, 0, path); + + Py_DECREF(path); + + if (nxt_slow_path(ret != 0)) { + nxt_alert(task, "Python failed to insert \"%V\" into \"sys.path\"", + &str); + return NXT_ERROR; + } + } + + return NXT_OK; +} + + static int nxt_python_init_threads(nxt_python_app_conf_t *c) { -- cgit From b7dba9006243f65e66d85a3a29841262c1c5dfef Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 23 Dec 2020 11:01:36 +0300 Subject: Static: fixing request memory pool leakage in router. When a static file larger than NXT_HTTP_STATIC_BUF_SIZE (128K) is served, two buffers are allocated and chained; each retains the whole request memory pool. Starting from 41331471eee7, the completion handler was called once for a linked buffer chain, but the second buffer got lost. This patch improves the completion handler's treatment of static buffers to handle all linked buffers. --- docs/changes.xml | 7 +++++++ src/nxt_http_static.c | 23 ++++++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index db7a2799..86bdc10a 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -61,6 +61,13 @@ had appeared in 1.19.0. + + +a memory leak occurring in the router process when serving a file larger than +128K; the bug had appeared in 1.13.0. + + + diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index 5687ef2c..df2655fc 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -395,12 +395,15 @@ static void nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) { ssize_t n, size; - nxt_buf_t *b, *fb; + nxt_buf_t *b, *fb, *next; nxt_off_t rest; nxt_http_request_t *r; b = obj; r = data; + +complete_buf: + fb = r->out; if (nxt_slow_path(fb == NULL || r->error)) { @@ -424,6 +427,8 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) goto clean; } + next = b->next; + if (n == rest) { nxt_file_close(task, fb->file); r->out = NULL; @@ -439,12 +444,24 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) b->mem.free = b->mem.pos + n; nxt_http_request_send(task, r, b); + + if (next != NULL) { + b = next; + goto complete_buf; + } + return; clean: - nxt_mp_free(r->mem_pool, b); - nxt_mp_release(r->mem_pool); + do { + next = b->next; + + nxt_mp_free(r->mem_pool, b); + nxt_mp_release(r->mem_pool); + + b = next; + } while (b != NULL); if (fb != NULL) { nxt_file_close(task, fb->file); -- cgit From 53af12def0623006509c08a3e081371a697acbd1 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 23 Dec 2020 18:30:17 +0300 Subject: Tests: ability to run unitd with specified "--user" option. --- test/conftest.py | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index c2a320de..d87e648b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -48,6 +48,11 @@ def pytest_addoption(parser): action="store_true", help="Run unsafe tests", ) + parser.addoption( + "--user", + type=str, + help="Default user for non-privileged processes of unitd", + ) unit_instance = {} @@ -60,6 +65,7 @@ def pytest_configure(config): option.print_log = config.option.print_log option.save_log = config.option.save_log option.unsafe = config.option.unsafe + option.user = config.option.user option.generated_tests = {} option.current_dir = os.path.abspath( @@ -283,26 +289,28 @@ def unit_run(): os.mkdir(temp_dir + '/state') + unitd_args = [ + unitd, + '--no-daemon', + '--modules', + build_dir, + '--state', + temp_dir + '/state', + '--pid', + temp_dir + '/unit.pid', + '--log', + temp_dir + '/unit.log', + '--control', + 'unix:' + temp_dir + '/control.unit.sock', + '--tmp', + temp_dir, + ] + + if option.user: + unitd_args.extend(['--user', option.user]) + with open(temp_dir + '/unit.log', 'w') as log: - unit_instance['process'] = subprocess.Popen( - [ - unitd, - '--no-daemon', - '--modules', - build_dir, - '--state', - temp_dir + '/state', - '--pid', - temp_dir + '/unit.pid', - '--log', - temp_dir + '/unit.log', - '--control', - 'unix:' + temp_dir + '/control.unit.sock', - '--tmp', - temp_dir, - ], - stderr=log, - ) + unit_instance['process'] = subprocess.Popen(unitd_args, stderr=log) if not waitforfiles(temp_dir + '/control.unit.sock'): _print_log() -- cgit From c99b175988d40fabf638c03962c01693e10b3795 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 23 Dec 2020 18:31:18 +0300 Subject: Packages: set the "--user" value to "nobody" for build-stage tests. This allows tests to pass well in cases when the default user specified in the ./configure parameters does not exist in the building environment. --- pkg/deb/Makefile | 4 ++-- pkg/rpm/Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index 6ce7373b..80898a8f 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -317,7 +317,7 @@ test: unit modules test -h debuild/unit-$(VERSION)/debian/build-unit/build/$${soname} || \ ln -fs `pwd`/$${so} debuild/unit-$(VERSION)/debian/build-unit/build/$${soname} ; \ done ; \ - ( cd debuild/unit-$(VERSION)/debian/build-unit && env python3 -m pytest $(PYTEST_ARGS) ) ; \ + ( cd debuild/unit-$(VERSION)/debian/build-unit && env python3 -m pytest --user=nobody $(PYTEST_ARGS) ) ; \ } test-debug: unit modules @@ -328,7 +328,7 @@ test-debug: unit modules test -h debuild/unit-$(VERSION)/debian/build-unit-debug/build/$${soname} || \ ln -fs `pwd`/$${so} debuild/unit-$(VERSION)/debian/build-unit-debug/build/$${soname} ; \ done ; \ - ( cd debuild/unit-$(VERSION)/debian/build-unit-debug && env python3 -m pytest $(PYTEST_ARGS) ) ; \ + ( cd debuild/unit-$(VERSION)/debian/build-unit-debug && env python3 -m pytest --user=nobody $(PYTEST_ARGS) ) ; \ } clean: diff --git a/pkg/rpm/Makefile b/pkg/rpm/Makefile index 1fefb262..80418149 100644 --- a/pkg/rpm/Makefile +++ b/pkg/rpm/Makefile @@ -276,7 +276,7 @@ test: unit modules test -h rpmbuild/BUILD/unit-$(VERSION)/build-nodebug/$${soname} || \ ln -fs `pwd`/$${so} rpmbuild/BUILD/unit-$(VERSION)/build-nodebug/$${soname} ; \ done ; \ - ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-nodebug build && env python3 -m pytest $(PYTEST_ARGS) ) ; \ + ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-nodebug build && env python3 -m pytest --user=nobody $(PYTEST_ARGS) ) ; \ } test-debug: unit modules @@ -287,7 +287,7 @@ test-debug: unit modules test -h rpmbuild/BUILD/unit-$(VERSION)/build-debug/$${soname} || \ ln -fs `pwd`/$${so} rpmbuild/BUILD/unit-$(VERSION)/build-debug/$${soname} ; \ done ; \ - ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-debug build && env python3 -m pytest $(PYTEST_ARGS) ) ; \ + ( cd rpmbuild/BUILD/unit-$(VERSION) && rm -f build && ln -s build-debug build && env python3 -m pytest --user=nobody $(PYTEST_ARGS) ) ; \ } clean: -- cgit From c369e25ae7e3de7759c5950a2a84870cc1a8401f Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Thu, 24 Dec 2020 15:18:49 +0300 Subject: Version bump for unit modules. This is required in order to build Debian packages from current tip without making manual interventions. Moving forward, this should be a part of every version bump commit. --- docs/changes.xml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/changes.xml b/docs/changes.xml index 86bdc10a..0e334419 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,28 @@ + + + + +NGINX Unit updated to 1.22.0. + + + + + + -- cgit From c981ac6558440f42670fc0a3d3857a3bb10e1f0a Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Thu, 24 Dec 2020 16:17:27 +0300 Subject: Packages: fixed an ability to override package version. This was broken since 00d8049418cf. --- pkg/deb/Makefile | 3 ++- pkg/docker/Makefile | 3 ++- pkg/rpm/Makefile | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index 80898a8f..16e73b94 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -2,9 +2,10 @@ include ../../version +DEFAULT_VERSION := $(NXT_VERSION) DEFAULT_RELEASE := 1 -VERSION ?= $(NXT_VERSION) +VERSION ?= $(DEFAULT_VERSION) RELEASE ?= $(DEFAULT_RELEASE) SRCDIR= unit-$(VERSION) diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile index 3b6c2720..455e9fb1 100644 --- a/pkg/docker/Makefile +++ b/pkg/docker/Makefile @@ -3,9 +3,10 @@ include ../../version include ../shasum.mak +DEFAULT_VERSION := $(NXT_VERSION) DEFAULT_RELEASE := 1 -VERSION ?= $(NXT_VERSION) +VERSION ?= $(DEFAULT_VERSION) RELEASE ?= $(DEFAULT_RELEASE) CODENAME := buster diff --git a/pkg/rpm/Makefile b/pkg/rpm/Makefile index 80418149..285e0857 100644 --- a/pkg/rpm/Makefile +++ b/pkg/rpm/Makefile @@ -2,9 +2,10 @@ include ../../version +DEFAULT_VERSION := $(NXT_VERSION) DEFAULT_RELEASE := 1 -VERSION ?= $(NXT_VERSION) +VERSION ?= $(DEFAULT_VERSION) RELEASE ?= $(DEFAULT_RELEASE) ifeq ($(shell test `rpm --eval '0%{?rhel} -eq 6 -a 0%{?amzn} -eq 0'`; echo $$?), 0) -- cgit From 3abca42caf44f03941545e5e92f35c0f329640e7 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Mon, 28 Dec 2020 12:51:30 +0300 Subject: Packages: fixed building for Ubuntu 16.04 "xenial". Changes introduced in a27532e3a17b effectively broke building of the unit package due to missed dh_installsystemd script in older debhelper 9.x. Once Ubuntu 16.04 reach EOL, the following actions should be made: - this commit should be reverted; - minimal debhelper version should be increased to 11. --- pkg/deb/debian/rules.in | 5 ++++ pkg/deb/debian/unit.init | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 pkg/deb/debian/unit.init diff --git a/pkg/deb/debian/rules.in b/pkg/deb/debian/rules.in index 1e76830f..aa7921d1 100644 --- a/pkg/deb/debian/rules.in +++ b/pkg/deb/debian/rules.in @@ -10,6 +10,7 @@ DPKG_EXPORT_BUILDFLAGS = 1 include /usr/share/dpkg/buildflags.mk DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) +CODENAME := $(shell lsb_release -cs) BUILDDIR_unit = $(CURDIR)/debian/build-unit BUILDDIR_unit_debug = $(CURDIR)/debian/build-unit-debug @@ -96,7 +97,11 @@ install: build do.tests dh_testroot dh_prep dh_installdirs +ifeq ($(CODENAME), xenial) + dh_installinit +else dh_installsystemd +endif dh_installlogrotate cd $(BUILDDIR_unit) && DESTDIR=$(INSTALLDIR) make install cd $(BUILDDIR_unit) && DESTDIR=$(INSTALLDIR_dev) make libunit-install diff --git a/pkg/deb/debian/unit.init b/pkg/deb/debian/unit.init new file mode 100644 index 00000000..900e97fd --- /dev/null +++ b/pkg/deb/debian/unit.init @@ -0,0 +1,74 @@ +#!/bin/sh +# +# unitd NGINX Unit +# +### BEGIN INIT INFO +# Provides: unitd +# Required-Start: $network $remote_fs +# Required-Stop: $network $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: NGINX Unit +# Description: NGINX Unit +### END INIT INFO +PATH=/sbin:/bin:/usr/sbin:/usr/bin +DAEMON=/usr/sbin/unitd +NAME=unit +DESC=unitd + +#includes lsb functions +. /lib/lsb/init-functions + +test -f $DAEMON || exit 0 + +umask 022 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + if start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \ + --exec $DAEMON -- $DAEMON_ARGS; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + status) + status_of_proc -p "$PIDFILE" "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + if start-stop-daemon --oknodo --stop --quiet --pidfile /var/run/$NAME.pid \ + --exec $DAEMON; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + reload|force-reload) + echo "Not implemented." >&2 + exit 1 + ;; + restart) + log_action_begin_msg "Restarting $DESC" "$NAME" + + start-stop-daemon --stop --quiet --pidfile \ + /var/run/$NAME.pid --exec $DAEMON || true + sleep 1 + if start-stop-daemon --start --quiet --pidfile \ + /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_ARGS; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + *) + echo "Usage: /etc/init.d/$NAME {start|status|stop|restart|reload|force-reload}" >&2 + exit 1 + ;; +esac + +exit 0 -- cgit From d3d6864bdc64f34924e686ff65da704b29aaaa93 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 29 Dec 2020 19:00:54 +0300 Subject: Node.js: ServerRequest and ServerResponse compliance to Stream API. ServerRequest now inherit stream Readable object. ServerResponse provides 'writable' property. Thanks to Wu Jian Ping (@wujjpp). This closes #274, closes #317 issues and closes #502 PR on GitHub. --- docs/changes.xml | 7 ++++ src/nodejs/unit-http/http_server.js | 50 +++++++++---------------- src/nodejs/unit-http/unit.cpp | 75 +++++++++++++++++++++++++++++++------ src/nodejs/unit-http/unit.h | 6 ++- test/test_node_application.py | 45 ---------------------- 5 files changed, 94 insertions(+), 89 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index 0e334419..b97cc823 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -37,6 +37,13 @@ ability to specify multiple directories in the "path" option of Python apps. + + +ServerRequest and ServerResponse objects of Node.js module are now compliant +to Stream API. + + + invalid HTTP responses were generated for some unusual status codes. diff --git a/src/nodejs/unit-http/http_server.js b/src/nodejs/unit-http/http_server.js index d378e410..e59296ae 100644 --- a/src/nodejs/unit-http/http_server.js +++ b/src/nodejs/unit-http/http_server.js @@ -11,6 +11,7 @@ const util = require('util'); const unit_lib = require('./build/Release/unit-http'); const Socket = require('./socket'); const WebSocketFrame = require('./websocket_frame'); +const Readable = require('stream').Readable; function ServerResponse(req) { @@ -23,6 +24,7 @@ function ServerResponse(req) { req._response = this; this.socket = req.socket; this.connection = req.connection; + this.writable = true; } util.inherits(ServerResponse, EventEmitter); @@ -268,6 +270,7 @@ ServerResponse.prototype._writeBody = function(chunk, encoding, callback) { res = this._write(chunk, 0, contentLength); if (res < contentLength) { this.socket.writable = false; + this.writable = false; o = new BufferedOutput(this, res, chunk, encoding, callback); this.server._output.push(o); @@ -328,6 +331,8 @@ ServerResponse.prototype.end = function end(chunk, encoding, callback) { if (typeof callback === 'function') { callback(); } + + this.emit("finish"); }); this.finished = true; @@ -337,15 +342,14 @@ ServerResponse.prototype.end = function end(chunk, encoding, callback) { }; function ServerRequest(server, socket) { - EventEmitter.call(this); + Readable.call(this); this.server = server; this.socket = socket; this.connection = socket; + this._pushed_eofchunk = false; } -util.inherits(ServerRequest, EventEmitter); - -ServerRequest.prototype.unpipe = undefined; +util.inherits(ServerRequest, Readable); ServerRequest.prototype.setTimeout = function setTimeout(msecs, callback) { this.timeout = msecs; @@ -377,35 +381,21 @@ ServerRequest.prototype.STATUS_CODES = function STATUS_CODES() { return http.STATUS_CODES; }; -ServerRequest.prototype.listeners = function listeners() { - return []; -}; - -ServerRequest.prototype.resume = function resume() { - return []; -}; +ServerRequest.prototype._request_read = unit_lib.request_read; -/* - * The "on" method is overridden to defer reading data until user code is - * ready, that is (ev === "data"). This can occur after req.emit("end") is - * executed, since the user code can be scheduled asynchronously by Promises - * and so on. Passing the data is postponed by process.nextTick() until - * the "on" method caller completes. - */ -ServerRequest.prototype.on = function on(ev, fn) { - Server.prototype.on.call(this, ev, fn); +ServerRequest.prototype._read = function _read(n) { + const b = this._request_read(n); - if (ev === "data") { - process.nextTick(function () { - if (this._data.length !== 0) { - this.emit("data", this._data); - } + if (b != null) { + this.push(b); + } - }.bind(this)); + if (!this._pushed_eofchunk && (b == null || b.length < n)) { + this._pushed_eofchunk = true; + this.push(null); } }; -ServerRequest.prototype.addListener = ServerRequest.prototype.on; function Server(requestListener) { EventEmitter.call(this); @@ -472,11 +462,6 @@ Server.prototype.emit_request = function (req, res) { } else { this.emit("request", req, res); } - - process.nextTick(() => { - req.emit("finish"); - req.emit("end"); - }); }; Server.prototype.emit_close = function () { @@ -523,6 +508,7 @@ Server.prototype.emit_drain = function () { } resp.socket.writable = true; + resp.writable = true; process.nextTick(() => { resp.emit("drain"); diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 0e049865..67b4377d 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -27,6 +27,7 @@ struct port_data_t { struct req_data_t { napi_ref sock_ref; + napi_ref req_ref; napi_ref resp_ref; napi_ref conn_ref; }; @@ -65,6 +66,7 @@ Unit::init(napi_env env, napi_value exports) constructor_ = napi.create_reference(ctor); napi.set_named_property(exports, "Unit", ctor); + napi.set_named_property(exports, "request_read", request_read); napi.set_named_property(exports, "response_send_headers", response_send_headers); napi.set_named_property(exports, "response_write", response_write); @@ -206,7 +208,7 @@ Unit::request_handler(nxt_unit_request_info_t *req) server_obj = get_server_object(); socket = create_socket(server_obj, req); - request = create_request(server_obj, socket); + request = create_request(server_obj, socket, req); response = create_response(server_obj, request, req); create_headers(req, request); @@ -301,6 +303,7 @@ Unit::close_handler(nxt_unit_request_info_t *req) nxt_napi::create(0)); remove_wrap(req_data->sock_ref); + remove_wrap(req_data->req_ref); remove_wrap(req_data->resp_ref); remove_wrap(req_data->conn_ref); @@ -488,9 +491,8 @@ Unit::get_server_object() void Unit::create_headers(nxt_unit_request_info_t *req, napi_value request) { - void *data; uint32_t i; - napi_value headers, raw_headers, buffer; + napi_value headers, raw_headers; napi_status status; nxt_unit_request_t *r; @@ -515,11 +517,6 @@ Unit::create_headers(nxt_unit_request_info_t *req, napi_value request) set_named_property(request, "url", r->target, r->target_length); set_named_property(request, "_websocket_handshake", r->websocket_handshake); - - buffer = create_buffer((size_t) req->content_length, &data); - nxt_unit_request_read(req, data, req->content_length); - - set_named_property(request, "_data", buffer); } @@ -577,13 +574,20 @@ Unit::create_socket(napi_value server_obj, nxt_unit_request_info_t *req) napi_value -Unit::create_request(napi_value server_obj, napi_value socket) +Unit::create_request(napi_value server_obj, napi_value socket, + nxt_unit_request_info_t *req) { - napi_value constructor; + napi_value constructor, res; + req_data_t *req_data; constructor = get_named_property(server_obj, "ServerRequest"); - return new_instance(constructor, server_obj, socket); + res = new_instance(constructor, server_obj, socket); + + req_data = (req_data_t *) req->data; + req_data->req_ref = wrap(res, req, req_destroy); + + return res; } @@ -642,6 +646,47 @@ Unit::create_websocket_frame(napi_value server_obj, } +napi_value +Unit::request_read(napi_env env, napi_callback_info info) +{ + void *data; + uint32_t wm; + nxt_napi napi(env); + napi_value this_arg, argv, buffer; + nxt_unit_request_info_t *req; + + try { + this_arg = napi.get_cb_info(info, argv); + + try { + req = napi.get_request_info(this_arg); + + } catch (exception &e) { + return nullptr; + } + + if (req->content_length == 0) { + return nullptr; + } + + wm = napi.get_value_uint32(argv); + + if (wm > req->content_length) { + wm = req->content_length; + } + + buffer = napi.create_buffer((size_t) wm, &data); + nxt_unit_request_read(req, data, wm); + + } catch (exception &e) { + napi.throw_error(e); + return nullptr; + } + + return buffer; +} + + napi_value Unit::response_send_headers(napi_env env, napi_callback_info info) { @@ -884,6 +929,7 @@ Unit::response_end(napi_env env, napi_callback_info info) req_data = (req_data_t *) req->data; napi.remove_wrap(req_data->sock_ref); + napi.remove_wrap(req_data->req_ref); napi.remove_wrap(req_data->resp_ref); napi.remove_wrap(req_data->conn_ref); @@ -1011,6 +1057,13 @@ Unit::sock_destroy(napi_env env, void *r, void *finalize_hint) } +void +Unit::req_destroy(napi_env env, void *r, void *finalize_hint) +{ + nxt_unit_req_debug(NULL, "req_destroy: %p", r); +} + + void Unit::resp_destroy(napi_env env, void *r, void *finalize_hint) { diff --git a/src/nodejs/unit-http/unit.h b/src/nodejs/unit-http/unit.h index 07823c26..4ef40d45 100644 --- a/src/nodejs/unit-http/unit.h +++ b/src/nodejs/unit-http/unit.h @@ -21,6 +21,7 @@ private: static void destroy(napi_env env, void *nativeObject, void *finalize_hint); static void conn_destroy(napi_env env, void *nativeObject, void *finalize_hint); static void sock_destroy(napi_env env, void *nativeObject, void *finalize_hint); + static void req_destroy(napi_env env, void *nativeObject, void *finalize_hint); static void resp_destroy(napi_env env, void *nativeObject, void *finalize_hint); static napi_value create_server(napi_env env, napi_callback_info info); @@ -50,7 +51,8 @@ private: napi_value create_socket(napi_value server_obj, nxt_unit_request_info_t *req); - napi_value create_request(napi_value server_obj, napi_value socket); + napi_value create_request(napi_value server_obj, napi_value socket, + nxt_unit_request_info_t *req); napi_value create_response(napi_value server_obj, napi_value request, nxt_unit_request_info_t *req); @@ -58,6 +60,8 @@ private: napi_value create_websocket_frame(napi_value server_obj, nxt_unit_websocket_frame_t *ws); + static napi_value request_read(napi_env env, napi_callback_info info); + static napi_value response_send_headers(napi_env env, napi_callback_info info); diff --git a/test/test_node_application.py b/test/test_node_application.py index f13a01e1..da29847a 100644 --- a/test/test_node_application.py +++ b/test/test_node_application.py @@ -222,22 +222,6 @@ class TestNodeApplication(TestApplicationNode): assert 'X-Header' not in headers, 'insensitive' assert 'X-header' not in headers, 'insensitive 2' - def test_node_application_promise_handler(self, temp_dir): - self.load('promise_handler') - - assert ( - self.post( - headers={ - 'Host': 'localhost', - 'Content-Type': 'text/html', - 'Connection': 'close', - }, - body='callback', - )['status'] - == 200 - ), 'promise handler request' - assert waitforfiles(temp_dir + '/node/callback'), 'promise handler' - def test_node_application_promise_handler_write_after_end(self): self.load('promise_handler') @@ -270,35 +254,6 @@ class TestNodeApplication(TestApplicationNode): ), 'promise end request' assert waitforfiles(temp_dir + '/node/callback'), 'promise end' - def test_node_application_promise_multiple_calls(self, temp_dir): - self.load('promise_handler') - - self.post( - headers={ - 'Host': 'localhost', - 'Content-Type': 'text/html', - 'Connection': 'close', - }, - body='callback1', - ) - - assert waitforfiles( - temp_dir + '/node/callback1' - ), 'promise first call' - - self.post( - headers={ - 'Host': 'localhost', - 'Content-Type': 'text/html', - 'Connection': 'close', - }, - body='callback2', - ) - - assert waitforfiles( - temp_dir + '/node/callback2' - ), 'promise second call' - @pytest.mark.skip('not yet') def test_node_application_header_name_valid(self): self.load('header_name_valid') -- cgit From d65a66f9d813294917822554311281c5e1a7126b Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Tue, 29 Dec 2020 19:01:24 +0300 Subject: Libunit: processing single port message. This partially reverts the optimisation introduced in 1d84b9e4b459 to avoid an unpredictable block in nxt_unit_process_port_msg(). Under high load, this function may never return control to its caller, and the external event loop (in Node.js and Python asyncio) won't be able to process other scheduled events. To reproduce the issue, two request processing types are needed: 'fast' and 'furious'. The 'fast' one simply returns a small response, while the 'furious' schedules asynchronous calls to external resources. Thus, if Unit is subjected to a large amount of 'fast' requests, the 'furious' request processing freezes until the high load ends. The issue was found by Wu Jian Ping (@wujjpp) during Node.js stream implementation discussion and relates to PR #502 on GitHub. --- src/nodejs/unit-http/unit.cpp | 176 +++++++++++++++++++++++++++++++----------- src/nxt_unit.c | 13 ---- src/python/nxt_python_asgi.c | 41 ++++++---- 3 files changed, 161 insertions(+), 69 deletions(-) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 67b4377d..589eca3f 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -13,15 +13,29 @@ #include -static void delete_port_data(uv_handle_t* handle); - napi_ref Unit::constructor_; struct port_data_t { - nxt_unit_ctx_t *ctx; - nxt_unit_port_t *port; - uv_poll_t poll; + port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p); + + void process_port_msg(); + void stop(); + + template + static port_data_t *get(T *handle); + + static void read_callback(uv_poll_t *handle, int status, int events); + static void timer_callback(uv_timer_t *handle); + static void delete_data(uv_handle_t* handle); + + nxt_unit_ctx_t *ctx; + nxt_unit_port_t *port; + uv_poll_t poll; + uv_timer_t timer; + int ref_count; + bool scheduled; + bool stopped; }; @@ -33,6 +47,106 @@ struct req_data_t { }; +port_data_t::port_data_t(nxt_unit_ctx_t *c, nxt_unit_port_t *p) : + ctx(c), port(p), ref_count(0), scheduled(false), stopped(false) +{ + timer.type = UV_UNKNOWN_HANDLE; +} + + +void +port_data_t::process_port_msg() +{ + int rc, err; + + rc = nxt_unit_process_port_msg(ctx, port); + + if (rc != NXT_UNIT_OK) { + return; + } + + if (timer.type == UV_UNKNOWN_HANDLE) { + err = uv_timer_init(poll.loop, &timer); + if (err < 0) { + nxt_unit_warn(ctx, "Failed to init uv.poll"); + return; + } + + ref_count++; + timer.data = this; + } + + if (!scheduled && !stopped) { + uv_timer_start(&timer, timer_callback, 0, 0); + + scheduled = true; + } +} + + +void +port_data_t::stop() +{ + stopped = true; + + uv_poll_stop(&poll); + + uv_close((uv_handle_t *) &poll, delete_data); + + if (timer.type == UV_UNKNOWN_HANDLE) { + return; + } + + uv_timer_stop(&timer); + + uv_close((uv_handle_t *) &timer, delete_data); +} + + +template +port_data_t * +port_data_t::get(T *handle) +{ + return (port_data_t *) handle->data; +} + + +void +port_data_t::read_callback(uv_poll_t *handle, int status, int events) +{ + get(handle)->process_port_msg(); +} + + +void +port_data_t::timer_callback(uv_timer_t *handle) +{ + port_data_t *data; + + data = get(handle); + + data->scheduled = false; + if (data->stopped) { + return; + } + + data->process_port_msg(); +} + + +void +port_data_t::delete_data(uv_handle_t* handle) +{ + port_data_t *data; + + data = get(handle); + + if (--data->ref_count <= 0) { + delete data; + } +} + + Unit::Unit(napi_env env, napi_value jsthis): nxt_napi(env), wrapper_(wrap(jsthis, this, destroy)), @@ -353,59 +467,50 @@ Unit::shm_ack_handler(nxt_unit_ctx_t *ctx) } -static void -nxt_uv_read_callback(uv_poll_t *handle, int status, int events) -{ - port_data_t *data; - - data = (port_data_t *) handle->data; - - nxt_unit_process_port_msg(data->ctx, data->port); -} - - int Unit::add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { - int err; - Unit *obj; - uv_loop_t *loop; - port_data_t *data; - napi_status status; + int err; + Unit *obj; + uv_loop_t *loop; + port_data_t *data; + napi_status status; if (port->in_fd != -1) { - obj = reinterpret_cast(ctx->unit->data); - if (fcntl(port->in_fd, F_SETFL, O_NONBLOCK) == -1) { nxt_unit_warn(ctx, "fcntl(%d, O_NONBLOCK) failed: %s (%d)", port->in_fd, strerror(errno), errno); return -1; } + obj = reinterpret_cast(ctx->unit->data); + status = napi_get_uv_event_loop(obj->env(), &loop); if (status != napi_ok) { nxt_unit_warn(ctx, "Failed to get uv.loop"); return NXT_UNIT_ERROR; } - data = new port_data_t; + data = new port_data_t(ctx, port); err = uv_poll_init(loop, &data->poll, port->in_fd); if (err < 0) { nxt_unit_warn(ctx, "Failed to init uv.poll"); + delete data; return NXT_UNIT_ERROR; } - err = uv_poll_start(&data->poll, UV_READABLE, nxt_uv_read_callback); + err = uv_poll_start(&data->poll, UV_READABLE, + port_data_t::read_callback); if (err < 0) { nxt_unit_warn(ctx, "Failed to start uv.poll"); + delete data; return NXT_UNIT_ERROR; } port->data = data; - data->ctx = ctx; - data->port = port; + data->ref_count++; data->poll.data = data; } @@ -421,26 +526,11 @@ Unit::remove_port(nxt_unit_t *unit, nxt_unit_port_t *port) if (port->data != NULL) { data = (port_data_t *) port->data; - if (data->port == port) { - uv_poll_stop(&data->poll); - - uv_close((uv_handle_t *) &data->poll, delete_port_data); - } + data->stop(); } } -static void -delete_port_data(uv_handle_t* handle) -{ - port_data_t *data; - - data = (port_data_t *) handle->data; - - delete data; -} - - void Unit::quit_cb(nxt_unit_ctx_t *ctx) { diff --git a/src/nxt_unit.c b/src/nxt_unit.c index 39e7f076..2fef17c5 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -5016,7 +5016,6 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; - nxt_unit_ctx_impl_t *ctx_impl; rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { @@ -5024,9 +5023,6 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) } lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - -retry: if (port == lib->shared_port) { rc = nxt_unit_shared_port_recv(ctx, port, rbuf); @@ -5052,15 +5048,6 @@ retry: nxt_unit_process_ready_req(ctx); - if (ctx_impl->online) { - rbuf = nxt_unit_read_buf_get(ctx); - if (nxt_slow_path(rbuf == NULL)) { - return NXT_UNIT_ERROR; - } - - goto retry; - } - return rc; } diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index 98aeedf4..a6f94507 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -1131,11 +1131,12 @@ nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx) static PyObject * nxt_py_asgi_port_read(PyObject *self, PyObject *args) { - int rc; - PyObject *arg; - Py_ssize_t n; - nxt_unit_ctx_t *ctx; - nxt_unit_port_t *port; + int rc; + PyObject *arg0, *arg1, *res; + Py_ssize_t n; + nxt_unit_ctx_t *ctx; + nxt_unit_port_t *port; + nxt_py_asgi_ctx_data_t *ctx_data; n = PyTuple_GET_SIZE(args); @@ -1147,31 +1148,45 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) return PyErr_Format(PyExc_TypeError, "invalid number of arguments"); } - arg = PyTuple_GET_ITEM(args, 0); - if (nxt_slow_path(arg == NULL || PyLong_Check(arg) == 0)) { + arg0 = PyTuple_GET_ITEM(args, 0); + if (nxt_slow_path(arg0 == NULL || PyLong_Check(arg0) == 0)) { return PyErr_Format(PyExc_TypeError, "the first argument is not a long"); } - ctx = PyLong_AsVoidPtr(arg); + ctx = PyLong_AsVoidPtr(arg0); - arg = PyTuple_GET_ITEM(args, 1); - if (nxt_slow_path(arg == NULL || PyLong_Check(arg) == 0)) { + arg1 = PyTuple_GET_ITEM(args, 1); + if (nxt_slow_path(arg1 == NULL || PyLong_Check(arg1) == 0)) { return PyErr_Format(PyExc_TypeError, "the second argument is not a long"); } - port = PyLong_AsVoidPtr(arg); - - nxt_unit_debug(ctx, "asgi_port_read %p %p", ctx, port); + port = PyLong_AsVoidPtr(arg1); rc = nxt_unit_process_port_msg(ctx, port); + nxt_unit_debug(ctx, "asgi_port_read(%p,%p): %d", ctx, port, rc); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return PyErr_Format(PyExc_RuntimeError, "error processing port %d message", port->id.id); } + if (rc == NXT_UNIT_OK) { + ctx_data = ctx->data; + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, + nxt_py_port_read, + arg0, arg1, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to call 'loop.call_soon'"); + nxt_python_print_exception(); + } + + Py_XDECREF(res); + } + Py_RETURN_NONE; } -- cgit From 13a06497def68d8157dfe2b6e0a7f7edcc2ef477 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Thu, 7 Jan 2021 16:51:27 +0000 Subject: Tests: reordered asserts to avoid a test race. The mount points are unmounted in the main process after it detects the app process died. By testing the `tmpfs: true` first, it happens that main could start the `tmpfs: false` test case before main cleans the old process mount points. --- test/test_go_isolation.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index 48c1b80c..0cd83056 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -342,16 +342,20 @@ class TestGoIsolation(TestApplicationGo): 'pid': True } + isolation['automount'] = { + 'tmpfs': False + } + self.load('ns_inspect', isolation=isolation) obj = self.getjson(url='/?mounts=true')['body'] assert ( - "/ /tmp" in obj['Mounts'] and "tmpfs" in obj['Mounts'] - ), 'app has /tmp mounted on /' + "/ /tmp" not in obj['Mounts'] and "tmpfs" not in obj['Mounts'] + ), 'app has no /tmp mounted' isolation['automount'] = { - 'tmpfs': False + 'tmpfs': True } self.load('ns_inspect', isolation=isolation) @@ -359,5 +363,5 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?mounts=true')['body'] assert ( - "/ /tmp" not in obj['Mounts'] and "tmpfs" not in obj['Mounts'] - ), 'app has no /tmp mounted' + "/ /tmp" in obj['Mounts'] and "tmpfs" in obj['Mounts'] + ), 'app has /tmp mounted on /' -- cgit From a0bc946db306b921fd2db909377e72bf6671e843 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Fri, 8 Jan 2021 10:38:46 +0000 Subject: Tests: fixed test_respawn.py to act upon test processes. Running `test_respawn_` test cases on a machine with Unit daemon in background would fail tests because `ps ax` was used without filtering out other unit instances. This patch also prevents from tests killing other Unit processes not related to tests. --- test/conftest.py | 4 ++++ test/test_respawn.py | 51 ++++++++++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d87e648b..0a4b541d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -443,6 +443,10 @@ def is_unsafe(request): def is_su(request): return os.geteuid() == 0 +@pytest.fixture +def unit_pid(request): + return unit_instance['process'].pid + def pytest_sessionfinish(session): unit_stop() shutil.rmtree(option.cache_dir) diff --git a/test/test_respawn.py b/test/test_respawn.py index eef2cb56..ed85ee95 100644 --- a/test/test_respawn.py +++ b/test/test_respawn.py @@ -21,17 +21,17 @@ class TestRespawn(TestApplicationPython): '1', 'applications/' + self.app_name + '/processes' ) - def pid_by_name(self, name): - output = subprocess.check_output(['ps', 'ax']).decode() - m = re.search(r'\s*(\d+).*' + name, output) - return m if m is None else m.group(1) + def pid_by_name(self, name, ppid): + output = subprocess.check_output(['ps', 'ax', '-O', 'ppid']).decode() + m = re.search(r'\s*(\d+)\s*' + str(ppid) + r'.*' + name, output) + return None if m is None else m.group(1) def kill_pids(self, *pids): subprocess.call(['kill', '-9'] + list(pids)) - def wait_for_process(self, process): + def wait_for_process(self, process, unit_pid): for i in range(50): - found = self.pid_by_name(process) + found = self.pid_by_name(process, unit_pid) if found is not None: break @@ -40,7 +40,10 @@ class TestRespawn(TestApplicationPython): return found - def smoke_test(self): + def find_proc(self, name, ppid, ps_output): + return re.findall(str(ppid) + r'.*' + name, ps_output) + + def smoke_test(self, unit_pid): for _ in range(5): assert 'success' in self.conf( '1', 'applications/' + self.app_name + '/processes' @@ -50,39 +53,41 @@ class TestRespawn(TestApplicationPython): # Check if the only one router, controller, # and application processes running. - output = subprocess.check_output(['ps', 'ax']).decode() - assert len(re.findall(self.PATTERN_ROUTER, output)) == 1 - assert len(re.findall(self.PATTERN_CONTROLLER, output)) == 1 - assert len(re.findall(self.app_name, output)) == 1 + out = subprocess.check_output(['ps', 'ax', '-O', 'ppid']).decode() + assert len(self.find_proc(self.PATTERN_ROUTER, unit_pid, out)) == 1 + assert len(self.find_proc(self.PATTERN_CONTROLLER, unit_pid, out)) == 1 + assert len(self.find_proc(self.app_name, unit_pid, out)) == 1 - def test_respawn_router(self, skip_alert): - pid = self.pid_by_name(self.PATTERN_ROUTER) + def test_respawn_router(self, skip_alert, unit_pid): + pid = self.pid_by_name(self.PATTERN_ROUTER, unit_pid) self.kill_pids(pid) skip_alert(r'process %s exited on signal 9' % pid) - assert self.wait_for_process(self.PATTERN_ROUTER) is not None + assert self.wait_for_process(self.PATTERN_ROUTER, unit_pid) is not None - self.smoke_test() + self.smoke_test(unit_pid) - def test_respawn_controller(self, skip_alert): - pid = self.pid_by_name(self.PATTERN_CONTROLLER) + def test_respawn_controller(self, skip_alert, unit_pid): + pid = self.pid_by_name(self.PATTERN_CONTROLLER, unit_pid) self.kill_pids(pid) skip_alert(r'process %s exited on signal 9' % pid) - assert self.wait_for_process(self.PATTERN_CONTROLLER) is not None + assert self.wait_for_process( + self.PATTERN_CONTROLLER, unit_pid + ) is not None assert self.get()['status'] == 200 - self.smoke_test() + self.smoke_test(unit_pid) - def test_respawn_application(self, skip_alert): - pid = self.pid_by_name(self.app_name) + def test_respawn_application(self, skip_alert, unit_pid): + pid = self.pid_by_name(self.app_name, unit_pid) self.kill_pids(pid) skip_alert(r'process %s exited on signal 9' % pid) - assert self.wait_for_process(self.app_name) is not None + assert self.wait_for_process(self.app_name, unit_pid) is not None - self.smoke_test() + self.smoke_test(unit_pid) -- cgit From f5ac1432463e58873b2c801b6db64dbe4d0e8f1c Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Tue, 12 Jan 2021 06:20:23 +0000 Subject: Tests: unit_stop() removed where possible. Since wait_for_record() was introduced there is no need to stop Unit before parsing unit.log. --- test/test_access_log.py | 31 ------------------------------- test/test_perl_application.py | 3 --- test/test_php_application.py | 16 ++++++++++------ test/test_python_application.py | 13 ------------- test/test_ruby_application.py | 12 ------------ test/test_usr1.py | 5 ----- 6 files changed, 10 insertions(+), 70 deletions(-) diff --git a/test/test_access_log.py b/test/test_access_log.py index 0a0a6633..e82e42e3 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -2,7 +2,6 @@ import time import pytest -from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.option import option @@ -50,8 +49,6 @@ class TestAccessLog(TestApplicationPython): body='0123456789', ) - unit_stop() - assert ( self.wait_for_record(r'"POST / HTTP/1.1" 200 10') is not None ), 'keepalive 2' @@ -78,8 +75,6 @@ Connection: close raw=True, ) - unit_stop() - assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "Referer-1" "-"') is not None @@ -100,8 +95,6 @@ Connection: close self.get(sock_type='ipv6') - unit_stop() - assert ( self.wait_for_record( r'::1 - - \[.+\] "GET / HTTP/1.1" 200 0 "-" "-"' @@ -120,8 +113,6 @@ Connection: close self.get(sock_type='unix', addr=addr) - unit_stop() - assert ( self.wait_for_record( r'unix: - - \[.+\] "GET / HTTP/1.1" 200 0 "-" "-"' @@ -140,8 +131,6 @@ Connection: close } ) - unit_stop() - assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "referer-value" "-"') is not None @@ -158,8 +147,6 @@ Connection: close } ) - unit_stop() - assert ( self.wait_for_record( r'"GET / HTTP/1.1" 200 0 "-" "user-agent-value"' @@ -172,8 +159,6 @@ Connection: close self.get(http_10=True) - unit_stop() - assert ( self.wait_for_record(r'"GET / HTTP/1.0" 200 0 "-" "-"') is not None ), 'http 1.0' @@ -187,8 +172,6 @@ Connection: close time.sleep(1) - unit_stop() - assert ( self.wait_for_record(r'"GE" 400 0 "-" "-"') is not None ), 'partial' @@ -200,8 +183,6 @@ Connection: close self.http(b"""GET /\n""", raw=True) - unit_stop() - assert ( self.wait_for_record(r'"GET /" 400 \d+ "-" "-"') is not None ), 'partial 2' @@ -215,8 +196,6 @@ Connection: close time.sleep(1) - unit_stop() - assert ( self.wait_for_record(r'"GET /" 400 0 "-" "-"') is not None ), 'partial 3' @@ -230,8 +209,6 @@ Connection: close time.sleep(1) - unit_stop() - assert ( self.wait_for_record(r'"GET / HTTP/1.1" 400 0 "-" "-"') is not None ), 'partial 4' @@ -244,8 +221,6 @@ Connection: close self.get(headers={'Connection': 'close'}) - unit_stop() - assert ( self.wait_for_record(r'"GET / HTTP/1.1" 400 \d+ "-" "-"') is not None @@ -256,8 +231,6 @@ Connection: close self.get(url='/?blah&var=val') - unit_stop() - assert ( self.wait_for_record( r'"GET /\?blah&var=val HTTP/1.1" 200 0 "-" "-"' @@ -272,8 +245,6 @@ Connection: close self.get(url='/delete') - unit_stop() - assert self.search_in_log(r'/delete', 'access.log') is None, 'delete' def test_access_log_change(self, temp_dir): @@ -285,8 +256,6 @@ Connection: close self.get() - unit_stop() - assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "-" "-"', 'new.log') is not None diff --git a/test/test_perl_application.py b/test/test_perl_application.py index ad91965f..e906aaca 100644 --- a/test/test_perl_application.py +++ b/test/test_perl_application.py @@ -2,7 +2,6 @@ import re import pytest -from conftest import unit_stop from unit.applications.lang.perl import TestApplicationPerl @@ -119,8 +118,6 @@ class TestPerlApplication(TestApplicationPerl): assert self.get()['body'] == '1', 'errors result' - unit_stop() - assert ( self.wait_for_record(r'\[error\].+Error in application') is not None diff --git a/test/test_php_application.py b/test/test_php_application.py index ad74faa8..9f0da531 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -2,10 +2,10 @@ import os import re import shutil import time +from subprocess import call import pytest -from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP from unit.option import option @@ -20,7 +20,7 @@ class TestPHPApplication(TestApplicationPHP): def set_opcache(self, app, val): assert 'success' in self.conf( - {"admin": {"opcache.enable": val, "opcache.enable_cli": val,},}, + {"admin": {"opcache.enable": val, "opcache.enable_cli": val}}, 'applications/' + app + '/options', ) @@ -99,7 +99,10 @@ class TestPHPApplication(TestApplicationPHP): assert self.get()['body'] == '0123' - unit_stop() + with open(temp_dir + '/unit.pid', 'r') as f: + pid = f.read().rstrip() + + call(['kill', '-s', 'USR1', pid]) with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: errs = re.findall(r'Error in fastcgi_finish_request', f.read()) @@ -113,7 +116,10 @@ class TestPHPApplication(TestApplicationPHP): assert resp['status'] == 200 assert resp['body'] == '' - unit_stop() + with open(temp_dir + '/unit.pid', 'r') as f: + pid = f.read().rstrip() + + call(['kill', '-s', 'USR1', pid]) with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: errs = re.findall(r'Error in fastcgi_finish_request', f.read()) @@ -538,8 +544,6 @@ class TestPHPApplication(TestApplicationPHP): assert self.get()['status'] == 200, 'status 2' - unit_stop() - pattern = r'\d{4}\/\d\d\/\d\d\s\d\d:.+\[notice\].+Error in application' assert self.wait_for_record(pattern) is not None, 'errors print' diff --git a/test/test_python_application.py b/test/test_python_application.py index b7cec831..01f53f8d 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -5,7 +5,6 @@ import time import pytest -from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.option import option @@ -157,8 +156,6 @@ custom-header: BLAH self.conf({"listeners": {}, "applications": {}}) - unit_stop() - assert ( self.wait_for_record(r'RuntimeError') is not None ), 'ctx iter atexit' @@ -337,8 +334,6 @@ Connection: close self.conf({"listeners": {}, "applications": {}}) - unit_stop() - assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' def test_python_process_switch(self): @@ -496,8 +491,6 @@ last line: 987654321 self.get() - unit_stop() - assert ( self.wait_for_record(r'\[error\].+Error in application\.') is not None @@ -537,8 +530,6 @@ last line: 987654321 self.get() - unit_stop() - assert self.wait_for_record(r'Close called\.') is not None, 'close' def test_python_application_close_error(self): @@ -546,8 +537,6 @@ last line: 987654321 self.get() - unit_stop() - assert ( self.wait_for_record(r'Close called\.') is not None ), 'close error' @@ -557,8 +546,6 @@ last line: 987654321 self.get() - unit_stop() - assert ( self.wait_for_record( r'\[error\].+the application returned not an iterable object' diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index 6a0c9c9f..8311f52f 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -2,7 +2,6 @@ import re import pytest -from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby @@ -175,8 +174,6 @@ class TestRubyApplication(TestApplicationRuby): self.get() - unit_stop() - assert ( self.wait_for_record(r'\[error\].+Error in application') is not None @@ -187,8 +184,6 @@ class TestRubyApplication(TestApplicationRuby): self.get() - unit_stop() - assert ( self.wait_for_record(r'\[error\].+1234567890') is not None ), 'errors puts int' @@ -198,8 +193,6 @@ class TestRubyApplication(TestApplicationRuby): self.get() - unit_stop() - assert ( self.wait_for_record(r'\[error\].+Error in application') is not None @@ -215,7 +208,6 @@ class TestRubyApplication(TestApplicationRuby): self.get() - unit_stop() assert ( self.wait_for_record(r'\[error\].+1234567890') is not None @@ -228,8 +220,6 @@ class TestRubyApplication(TestApplicationRuby): self.conf({"listeners": {}, "applications": {}}) - unit_stop() - assert ( self.wait_for_record(r'\[error\].+At exit called\.') is not None ), 'at exit' @@ -289,8 +279,6 @@ class TestRubyApplication(TestApplicationRuby): assert self.get()['status'] == 500, 'body each error status' - unit_stop() - assert ( self.wait_for_record(r'\[error\].+Failed to run ruby script') is not None diff --git a/test/test_usr1.py b/test/test_usr1.py index 44f19d23..dbb5265c 100644 --- a/test/test_usr1.py +++ b/test/test_usr1.py @@ -1,7 +1,6 @@ import os from subprocess import call -from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.utils import waitforfiles @@ -41,8 +40,6 @@ class TestUSR1(TestApplicationPython): assert self.get(url='/usr1')['status'] == 200 - unit_stop() - assert ( self.wait_for_record(r'"GET /usr1 HTTP/1.1" 200 0 "-" "-"', log) is not None @@ -74,8 +71,6 @@ class TestUSR1(TestApplicationPython): body = 'body_for_a_log_unit' assert self.post(body=body)['status'] == 200 - unit_stop() - assert self.wait_for_record(body) is not None, 'rename new' assert self.search_in_log(body, log_new) is None, 'rename new 2' -- cgit From 6dc9c47ccd26b23b61b7522803a667c2e515e260 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 13 Jan 2021 06:22:43 +0000 Subject: Tests: style. --- test/conftest.py | 4 +--- test/test_access_log.py | 1 - test/test_asgi_application.py | 1 - test/test_asgi_lifespan.py | 1 - test/test_asgi_websockets.py | 1 - test/test_configuration.py | 1 - test/test_go_isolation.py | 2 -- test/test_go_isolation_rootfs.py | 1 - test/test_http_header.py | 1 - test/test_java_isolation_rootfs.py | 1 - test/test_java_websockets.py | 1 - test/test_node_application.py | 1 - test/test_node_websockets.py | 1 - test/test_perl_application.py | 1 - test/test_php_application.py | 1 - test/test_php_isolation.py | 3 --- test/test_proxy.py | 1 - test/test_python_application.py | 1 - test/test_python_isolation.py | 2 -- test/test_python_isolation_chroot.py | 1 - test/test_python_procman.py | 1 - test/test_routing.py | 1 - test/test_ruby_application.py | 1 - test/test_ruby_isolation.py | 3 +-- test/test_settings.py | 2 -- test/test_static.py | 1 - test/test_tls.py | 1 - 27 files changed, 2 insertions(+), 35 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 0a4b541d..b36aabad 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,7 +4,6 @@ import platform import re import shutil import signal -import socket import stat import subprocess import sys @@ -13,11 +12,10 @@ import time from multiprocessing import Process import pytest - from unit.check.go import check_go +from unit.check.isolation import check_isolation from unit.check.node import check_node from unit.check.tls import check_openssl -from unit.check.isolation import check_isolation from unit.option import option from unit.utils import public_dir from unit.utils import waitforfiles diff --git a/test/test_access_log.py b/test/test_access_log.py index e82e42e3..045473ae 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -1,7 +1,6 @@ import time import pytest - from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 9f4b70a4..5770265d 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -3,7 +3,6 @@ import time from distutils.version import LooseVersion import pytest - from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index 43286e22..3ecece43 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -2,7 +2,6 @@ import os from distutils.version import LooseVersion import pytest - from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 76cd8e80..7c9ec555 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -3,7 +3,6 @@ import time from distutils.version import LooseVersion import pytest - from unit.applications.lang.python import TestApplicationPython from unit.applications.websockets import TestApplicationWebsocket from unit.option import option diff --git a/test/test_configuration.py b/test/test_configuration.py index 7feb3adb..b7417264 100644 --- a/test/test_configuration.py +++ b/test/test_configuration.py @@ -1,5 +1,4 @@ import pytest - from unit.control import TestControl diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index 0cd83056..e8fa26c6 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -1,10 +1,8 @@ import grp import os import pwd -import shutil import pytest - from unit.applications.lang.go import TestApplicationGo from unit.option import option from unit.utils import getns diff --git a/test/test_go_isolation_rootfs.py b/test/test_go_isolation_rootfs.py index 1cc59c67..2bded5ec 100644 --- a/test/test_go_isolation_rootfs.py +++ b/test/test_go_isolation_rootfs.py @@ -1,7 +1,6 @@ import os import pytest - from unit.applications.lang.go import TestApplicationGo diff --git a/test/test_http_header.py b/test/test_http_header.py index fdb557cf..ca355eb7 100644 --- a/test/test_http_header.py +++ b/test/test_http_header.py @@ -1,5 +1,4 @@ import pytest - from unit.applications.lang.python import TestApplicationPython diff --git a/test/test_java_isolation_rootfs.py b/test/test_java_isolation_rootfs.py index a401e23b..91773981 100644 --- a/test/test_java_isolation_rootfs.py +++ b/test/test_java_isolation_rootfs.py @@ -2,7 +2,6 @@ import os import subprocess import pytest - from unit.applications.lang.java import TestApplicationJava from unit.option import option diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 729aa31d..315c496d 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -2,7 +2,6 @@ import struct import time import pytest - from unit.applications.lang.java import TestApplicationJava from unit.applications.websockets import TestApplicationWebsocket from unit.option import option diff --git a/test/test_node_application.py b/test/test_node_application.py index da29847a..b277dc3a 100644 --- a/test/test_node_application.py +++ b/test/test_node_application.py @@ -1,7 +1,6 @@ import re import pytest - from unit.applications.lang.node import TestApplicationNode from unit.utils import waitforfiles diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index d7444bf7..4f1e5906 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -2,7 +2,6 @@ import struct import time import pytest - from unit.applications.lang.node import TestApplicationNode from unit.applications.websockets import TestApplicationWebsocket from unit.option import option diff --git a/test/test_perl_application.py b/test/test_perl_application.py index e906aaca..dfd8be6c 100644 --- a/test/test_perl_application.py +++ b/test/test_perl_application.py @@ -1,7 +1,6 @@ import re import pytest - from unit.applications.lang.perl import TestApplicationPerl diff --git a/test/test_php_application.py b/test/test_php_application.py index 9f0da531..981d9e95 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -5,7 +5,6 @@ import time from subprocess import call import pytest - from unit.applications.lang.php import TestApplicationPHP from unit.option import option diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index b0fea383..2e458023 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -1,7 +1,4 @@ -import shutil - import pytest - from unit.applications.lang.php import TestApplicationPHP from unit.option import option diff --git a/test/test_proxy.py b/test/test_proxy.py index 0eebab1e..b7c34390 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -3,7 +3,6 @@ import socket import time import pytest - from conftest import run_process from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_python_application.py b/test/test_python_application.py index 01f53f8d..5aa60712 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -4,7 +4,6 @@ import re import time import pytest - from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index ad830269..aa96af51 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -1,7 +1,5 @@ -import shutil import pytest - from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_python_isolation_chroot.py b/test/test_python_isolation_chroot.py index 7f559bcc..7281f5f6 100644 --- a/test/test_python_isolation_chroot.py +++ b/test/test_python_isolation_chroot.py @@ -1,5 +1,4 @@ import pytest - from unit.applications.lang.python import TestApplicationPython diff --git a/test/test_python_procman.py b/test/test_python_procman.py index 42a197a7..bfc218a9 100644 --- a/test/test_python_procman.py +++ b/test/test_python_procman.py @@ -3,7 +3,6 @@ import subprocess import time import pytest - from unit.applications.lang.python import TestApplicationPython from unit.option import option diff --git a/test/test_routing.py b/test/test_routing.py index 30f25a9c..21c1899e 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import pytest - from unit.applications.proto import TestApplicationProto from unit.option import option diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index 8311f52f..a84b2990 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -1,7 +1,6 @@ import re import pytest - from unit.applications.lang.ruby import TestApplicationRuby diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index fab428e9..c49b6732 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -1,8 +1,7 @@ +import os import shutil -import os import pytest - from unit.applications.lang.ruby import TestApplicationRuby from unit.option import option diff --git a/test/test_settings.py b/test/test_settings.py index 22830a3b..c59ca8b9 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -3,7 +3,6 @@ import socket import time import pytest - from unit.applications.lang.python import TestApplicationPython @@ -297,4 +296,3 @@ Connection: close assert bool(resp), 'response from application 4' assert resp['status'] == 200, 'status 4' assert resp['body'] == body, 'body 4' - diff --git a/test/test_static.py b/test/test_static.py index 4591aa03..3e85b435 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -2,7 +2,6 @@ import os import socket import pytest - from unit.applications.proto import TestApplicationProto from unit.option import option from unit.utils import waitforfiles diff --git a/test/test_tls.py b/test/test_tls.py index 5a408ee2..f5df5ee1 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -4,7 +4,6 @@ import ssl import subprocess import pytest - from unit.applications.tls import TestApplicationTLS from unit.option import option -- cgit From db9b70932b2e3e4df8b4bcdb24fedce042da15aa Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 13 Jan 2021 06:24:26 +0000 Subject: Tests: waitformount() and waitforunmount() introduced. --- test/unit/utils.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/unit/utils.py b/test/unit/utils.py index 1307a4f6..7a0a3fe5 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -1,5 +1,6 @@ import os import socket +import subprocess import time import pytest @@ -49,6 +50,37 @@ def waitforsocket(port): pytest.fail('Can\'t connect to the 127.0.0.1:' + port) +def findmnt(): + try: + out = subprocess.check_output( + ['findmnt', '--raw'], stderr=subprocess.STDOUT + ).decode() + except FileNotFoundError: + pytest.skip('requires findmnt') + + return out + + +def waitformount(template, wait=50): + for i in range(wait): + if findmnt().find(template) != -1: + return True + + time.sleep(0.1) + + return False + + +def waitforunmount(template, wait=50): + for i in range(wait): + if findmnt().find(template) == -1: + return True + + time.sleep(0.1) + + return False + + def getns(nstype): # read namespace id from symlink file: # it points to: ':[]' -- cgit From 22dac5bd62d19c0f8743b63ec8c47b3b071eb000 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 13 Jan 2021 06:24:32 +0000 Subject: Tests: "language_deps" option checked more carefully. --- test/test_python_isolation.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index aa96af51..e1a42c98 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -2,6 +2,9 @@ import pytest from unit.applications.lang.python import TestApplicationPython from unit.option import option +from unit.utils import findmnt +from unit.utils import waitformount +from unit.utils import waitforunmount class TestPythonIsolation(TestApplicationPython): @@ -60,39 +63,28 @@ class TestPythonIsolation(TestApplicationPython): ), 'application exists in rootfs' def test_python_isolation_rootfs_no_language_deps(self, is_su, temp_dir): - isolation_features = option.available['features']['isolation'].keys() - if not is_su: - if not 'unprivileged_userns_clone' in isolation_features: - pytest.skip('requires unprivileged userns or root') - - if 'user' not in isolation_features: - pytest.skip('user namespace is not supported') - - if 'mnt' not in isolation_features: - pytest.skip('mnt namespace is not supported') - - if 'pid' not in isolation_features: - pytest.skip('pid namespace is not supported') + pytest.skip('requires root') isolation = { 'rootfs': temp_dir, 'automount': {'language_deps': False} } - if not is_su: - isolation['namespaces'] = { - 'mount': True, - 'credential': True, - 'pid': True - } - self.load('empty', isolation=isolation) + assert findmnt().find(temp_dir) == -1 assert (self.get()['status'] != 200), 'disabled language_deps' + assert findmnt().find(temp_dir) == -1 isolation['automount']['language_deps'] = True self.load('empty', isolation=isolation) + assert findmnt().find(temp_dir) == -1 assert (self.get()['status'] == 200), 'enabled language_deps' + assert waitformount(temp_dir), 'language_deps mount' + + self.conf({"listeners": {}, "applications": {}}) + + assert waitforunmount(temp_dir), 'language_deps unmount' -- cgit From 5d983ea762eba5fe26a07defbc09eeec8ecc5aeb Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Wed, 13 Jan 2021 06:24:41 +0000 Subject: Tests: added test for "procfs" option. --- test/test_python_isolation.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index e1a42c98..680f2c03 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -88,3 +88,26 @@ class TestPythonIsolation(TestApplicationPython): self.conf({"listeners": {}, "applications": {}}) assert waitforunmount(temp_dir), 'language_deps unmount' + + def test_python_isolation_procfs(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() + + if not is_su: + pytest.skip('requires root') + + isolation = {'rootfs': temp_dir, 'automount': {'procfs': False}} + + self.load('ns_inspect', isolation=isolation) + + assert ( + self.getjson(url='/?path=/proc/self')['body']['FileExists'] + == False + ), 'no /proc/self' + + isolation['automount']['procfs'] = True + + self.load('ns_inspect', isolation=isolation) + + assert ( + self.getjson(url='/?path=/proc/self')['body']['FileExists'] == True + ), '/proc/self' -- cgit From d43a84139d1adedbae8def67c8bbee09d8cf4581 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 14 Jan 2021 03:04:20 +0000 Subject: Tests: added missing checks for configuration results. --- test/test_access_log.py | 12 ++++++++---- test/test_go_application.py | 8 ++++---- test/test_php_application.py | 30 +++++++++++++++--------------- test/test_php_basic.py | 26 +++++++++++++++----------- test/test_proxy.py | 6 ++++-- test/test_python_application.py | 4 ++-- test/test_python_basic.py | 24 ++++++++++++++---------- test/test_python_procman.py | 2 +- test/test_routing.py | 2 +- test/test_ruby_application.py | 2 +- test/test_tls.py | 12 +++++++----- 11 files changed, 72 insertions(+), 56 deletions(-) diff --git a/test/test_access_log.py b/test/test_access_log.py index 045473ae..65d5e50a 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -90,7 +90,9 @@ Connection: close def test_access_log_ipv6(self): self.load('empty') - self.conf({"[::1]:7080": {"pass": "applications/empty"}}, 'listeners') + assert 'success' in self.conf( + {"[::1]:7080": {"pass": "applications/empty"}}, 'listeners' + ) self.get(sock_type='ipv6') @@ -106,7 +108,7 @@ Connection: close addr = option.temp_dir + '/sock' - self.conf( + assert 'success' in self.conf( {"unix:" + addr: {"pass": "applications/empty"}}, 'listeners' ) @@ -240,7 +242,7 @@ Connection: close def test_access_log_delete(self): self.load('empty') - self.conf_delete('access_log') + assert 'success' in self.conf_delete('access_log') self.get(url='/delete') @@ -251,7 +253,9 @@ Connection: close self.get() - self.conf('"' + option.temp_dir + '/new.log"', 'access_log') + assert 'success' in self.conf( + '"' + option.temp_dir + '/new.log"', 'access_log' + ) self.get() diff --git a/test/test_go_application.py b/test/test_go_application.py index 8c77dfc5..e833d190 100644 --- a/test/test_go_application.py +++ b/test/test_go_application.py @@ -149,7 +149,7 @@ class TestGoApplication(TestApplicationGo): arg2 = '--cc-opt=\'-O0 -DNXT_DEBUG_MEMORY=1 -fsanitize=address\'' arg3 = '--debug' - self.conf( + assert 'success' in self.conf( '["' + arg1 + '", "' + arg2 + '", "' + arg3 + '"]', 'applications/command_line_arguments/arguments', ) @@ -163,15 +163,15 @@ class TestGoApplication(TestApplicationGo): args_path = 'applications/command_line_arguments/arguments' - self.conf('["0", "a", "$", ""]', args_path) + assert 'success' in self.conf('["0", "a", "$", ""]', args_path) assert self.get()['body'] == '0,a,$,', 'arguments' - self.conf('["-1", "b", "%"]', args_path) + assert 'success' in self.conf('["-1", "b", "%"]', args_path) assert self.get()['body'] == '-1,b,%', 'arguments change' - self.conf('[]', args_path) + assert 'success' in self.conf('[]', args_path) assert ( self.get()['headers']['Content-Length'] == '0' diff --git a/test/test_php_application.py b/test/test_php_application.py index 981d9e95..e73c67ba 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -268,7 +268,7 @@ class TestPHPApplication(TestApplicationPHP): assert self.get()['headers']['X-Precision'] != '4', 'ini value default' - self.conf( + assert 'success' in self.conf( {"file": "ini/php.ini"}, 'applications/ini_precision/options' ) @@ -290,7 +290,7 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_ini_admin(self): self.load('ini_precision') - self.conf( + assert 'success' in self.conf( {"file": "php.ini", "admin": {"precision": "5"}}, 'applications/ini_precision/options', ) @@ -300,7 +300,7 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_ini_user(self): self.load('ini_precision') - self.conf( + assert 'success' in self.conf( {"file": "php.ini", "user": {"precision": "5"}}, 'applications/ini_precision/options', ) @@ -310,13 +310,13 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_ini_user_2(self): self.load('ini_precision') - self.conf( + assert 'success' in self.conf( {"file": "ini/php.ini"}, 'applications/ini_precision/options' ) assert self.get()['headers']['X-Precision'] == '4', 'ini user file' - self.conf( + assert 'success' in self.conf( {"precision": "5"}, 'applications/ini_precision/options/user' ) @@ -325,7 +325,7 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_ini_set_admin(self): self.load('ini_precision') - self.conf( + assert 'success' in self.conf( {"admin": {"precision": "5"}}, 'applications/ini_precision/options' ) @@ -336,7 +336,7 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_ini_set_user(self): self.load('ini_precision') - self.conf( + assert 'success' in self.conf( {"user": {"precision": "5"}}, 'applications/ini_precision/options' ) @@ -347,7 +347,7 @@ class TestPHPApplication(TestApplicationPHP): def test_php_application_ini_repeat(self): self.load('ini_precision') - self.conf( + assert 'success' in self.conf( {"user": {"precision": "5"}}, 'applications/ini_precision/options' ) @@ -360,7 +360,7 @@ class TestPHPApplication(TestApplicationPHP): self.before_disable_functions() - self.conf( + assert 'success' in self.conf( {"admin": {"disable_functions": "exec"}}, 'applications/time_exec/options', ) @@ -375,7 +375,7 @@ class TestPHPApplication(TestApplicationPHP): self.before_disable_functions() - self.conf( + assert 'success' in self.conf( {"admin": {"disable_functions": "exec,time"}}, 'applications/time_exec/options', ) @@ -452,7 +452,7 @@ class TestPHPApplication(TestApplicationPHP): self.before_disable_functions() - self.conf( + assert 'success' in self.conf( {"admin": {"disable_functions": "exec time"}}, 'applications/time_exec/options', ) @@ -471,7 +471,7 @@ class TestPHPApplication(TestApplicationPHP): self.before_disable_functions() - self.conf( + assert 'success' in self.conf( {"user": {"disable_functions": "exec"}}, 'applications/time_exec/options', ) @@ -488,7 +488,7 @@ class TestPHPApplication(TestApplicationPHP): self.before_disable_functions() - self.conf( + assert 'success' in self.conf( {"admin": {"disable_functions": "blah"}}, 'applications/time_exec/options', ) @@ -509,7 +509,7 @@ class TestPHPApplication(TestApplicationPHP): r'012345', self.get()['body'] ), 'disable_classes before' - self.conf( + assert 'success' in self.conf( {"admin": {"disable_classes": "DateTime"}}, 'applications/date_time/options', ) @@ -525,7 +525,7 @@ class TestPHPApplication(TestApplicationPHP): r'012345', self.get()['body'] ), 'disable_classes before' - self.conf( + assert 'success' in self.conf( {"user": {"disable_classes": "DateTime"}}, 'applications/date_time/options', ) diff --git a/test/test_php_basic.py b/test/test_php_basic.py index 1420ec21..bcd66173 100644 --- a/test/test_php_basic.py +++ b/test/test_php_basic.py @@ -19,7 +19,7 @@ class TestPHPBasic(TestControl): } def test_php_get_applications(self): - self.conf(self.conf_app, 'applications') + assert 'success' in self.conf(self.conf_app, 'applications') conf = self.conf_get() @@ -55,7 +55,7 @@ class TestPHPBasic(TestControl): ), 'spare processes' def test_php_get_listeners(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) assert self.conf_get()['listeners'] == { "*:7080": {"pass": "applications/app"} @@ -70,16 +70,20 @@ class TestPHPBasic(TestControl): }, 'listeners prefix 2' def test_php_change_listener(self): - self.conf(self.conf_basic) - self.conf({"*:7081": {"pass": "applications/app"}}, 'listeners') + assert 'success' in self.conf(self.conf_basic) + assert 'success' in self.conf( + {"*:7081": {"pass": "applications/app"}}, 'listeners' + ) assert self.conf_get('listeners') == { "*:7081": {"pass": "applications/app"} }, 'change listener' def test_php_add_listener(self): - self.conf(self.conf_basic) - self.conf({"pass": "applications/app"}, 'listeners/*:7082') + assert 'success' in self.conf(self.conf_basic) + assert 'success' in self.conf( + {"pass": "applications/app"}, 'listeners/*:7082' + ) assert self.conf_get('listeners') == { "*:7080": {"pass": "applications/app"}, @@ -87,20 +91,20 @@ class TestPHPBasic(TestControl): }, 'add listener' def test_php_change_application(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) - self.conf('30', 'applications/app/processes/max') + assert 'success' in self.conf('30', 'applications/app/processes/max') assert ( self.conf_get('applications/app/processes/max') == 30 ), 'change application max' - self.conf('"/www"', 'applications/app/root') + assert 'success' in self.conf('"/www"', 'applications/app/root') assert ( self.conf_get('applications/app/root') == '/www' ), 'change application root' def test_php_delete(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) assert 'error' in self.conf_delete('applications/app') assert 'success' in self.conf_delete('listeners/*:7080') @@ -108,7 +112,7 @@ class TestPHPBasic(TestControl): assert 'error' in self.conf_delete('applications/app') def test_php_delete_blocks(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) assert 'success' in self.conf_delete('listeners') assert 'success' in self.conf_delete('applications') diff --git a/test/test_proxy.py b/test/test_proxy.py index b7c34390..2d305e98 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -172,7 +172,9 @@ Content-Length: 10 assert resp['status'] == 200, 'status' assert resp['body'] == payload, 'body' - self.conf({'http': {'max_body_size': 32 * 1024 * 1024}}, 'settings') + assert 'success' in self.conf( + {'http': {'max_body_size': 32 * 1024 * 1024}}, 'settings' + ) payload = '0123456789abcdef' * 32 * 64 * 1024 resp = self.post_http10(body=payload, read_buffer_size=1024 * 1024) @@ -486,7 +488,7 @@ Content-Length: 10 r'accept.*failed', r'new connections are not accepted', ) - self.conf( + assert 'success' in self.conf( { "listeners": { "*:7080": {"pass": "routes"}, diff --git a/test/test_python_application.py b/test/test_python_application.py index 5aa60712..5ad0901d 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -153,7 +153,7 @@ custom-header: BLAH assert resp['status'] == 200, 'ctx iter status' assert resp['body'] == '0123456789', 'ctx iter body' - self.conf({"listeners": {}, "applications": {}}) + assert 'success' in self.conf({"listeners": {}, "applications": {}}) assert ( self.wait_for_record(r'RuntimeError') is not None @@ -331,7 +331,7 @@ Connection: close self.get() - self.conf({"listeners": {}, "applications": {}}) + assert 'success' in self.conf({"listeners": {}, "applications": {}}) assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' diff --git a/test/test_python_basic.py b/test/test_python_basic.py index 0cc70e51..e661a89c 100644 --- a/test/test_python_basic.py +++ b/test/test_python_basic.py @@ -58,7 +58,7 @@ class TestPythonBasic(TestControl): assert self.conf_get('applications/app/processes/spare') == 0, 'spare' def test_python_get_listeners(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) assert self.conf_get()['listeners'] == { "*:7080": {"pass": "applications/app"} @@ -73,16 +73,20 @@ class TestPythonBasic(TestControl): }, 'listeners prefix 2' def test_python_change_listener(self): - self.conf(self.conf_basic) - self.conf({"*:7081": {"pass": "applications/app"}}, 'listeners') + assert 'success' in self.conf(self.conf_basic) + assert 'success' in self.conf( + {"*:7081": {"pass": "applications/app"}}, 'listeners' + ) assert self.conf_get('listeners') == { "*:7081": {"pass": "applications/app"} }, 'change listener' def test_python_add_listener(self): - self.conf(self.conf_basic) - self.conf({"pass": "applications/app"}, 'listeners/*:7082') + assert 'success' in self.conf(self.conf_basic) + assert 'success' in self.conf( + {"pass": "applications/app"}, 'listeners/*:7082' + ) assert self.conf_get('listeners') == { "*:7080": {"pass": "applications/app"}, @@ -90,20 +94,20 @@ class TestPythonBasic(TestControl): }, 'add listener' def test_python_change_application(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) - self.conf('30', 'applications/app/processes/max') + assert 'success' in self.conf('30', 'applications/app/processes/max') assert ( self.conf_get('applications/app/processes/max') == 30 ), 'change application max' - self.conf('"/www"', 'applications/app/path') + assert 'success' in self.conf('"/www"', 'applications/app/path') assert ( self.conf_get('applications/app/path') == '/www' ), 'change application path' def test_python_delete(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) assert 'error' in self.conf_delete('applications/app') assert 'success' in self.conf_delete('listeners/*:7080') @@ -111,7 +115,7 @@ class TestPythonBasic(TestControl): assert 'error' in self.conf_delete('applications/app') def test_python_delete_blocks(self): - self.conf(self.conf_basic) + assert 'success' in self.conf(self.conf_basic) assert 'success' in self.conf_delete('listeners') assert 'success' in self.conf_delete('applications') diff --git a/test/test_python_procman.py b/test/test_python_procman.py index bfc218a9..ac403ce4 100644 --- a/test/test_python_procman.py +++ b/test/test_python_procman.py @@ -197,6 +197,6 @@ class TestPythonProcman(TestApplicationPython): ), 'max zero' def stop_all(self): - self.conf({"listeners": {}, "applications": {}}) + assert 'success' in self.conf({"listeners": {}, "applications": {}}) assert len(self.pids_for_process()) == 0, 'stop all' diff --git a/test/test_routing.py b/test/test_routing.py index 21c1899e..4d27cb61 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -316,7 +316,7 @@ class TestRouting(TestApplicationProto): check_pass_error("%1", "%1") def test_routes_absent(self): - self.conf( + assert 'success' in self.conf( { "listeners": {"*:7081": {"pass": "applications/empty"}}, "applications": { diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index a84b2990..b18a6cee 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -217,7 +217,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.conf({"listeners": {}, "applications": {}}) + assert 'success' in self.conf({"listeners": {}, "applications": {}}) assert ( self.wait_for_record(r'\[error\].+At exit called\.') is not None diff --git a/test/test_tls.py b/test/test_tls.py index f5df5ee1..89c57d07 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -19,7 +19,7 @@ class TestTLS(TestApplicationTLS): return self.date_to_sec_epoch(date, '%b %d %H:%M:%S %Y %Z') def add_tls(self, application='empty', cert='default', port=7080): - self.conf( + assert 'success' in self.conf( { "pass": "applications/" + application, "tls": {"certificate": cert} @@ -28,7 +28,7 @@ class TestTLS(TestApplicationTLS): ) def remove_tls(self, application='empty', port=7080): - self.conf( + assert 'success' in self.conf( {"pass": "applications/" + application}, 'listeners/*:' + str(port) ) @@ -477,8 +477,10 @@ basicConstraints = critical,CA:TRUE""" read_timeout=1, ) - self.conf({"pass": "applications/empty"}, 'listeners/*:7080') - self.conf_delete('/certificates/default') + assert 'success' in self.conf( + {"pass": "applications/empty"}, 'listeners/*:7080' + ) + assert 'success' in self.conf_delete('/certificates/default') try: resp = self.get_ssl( @@ -508,7 +510,7 @@ basicConstraints = critical,CA:TRUE""" self.certificate() - self.conf('1', 'applications/mirror/processes') + assert 'success' in self.conf('1', 'applications/mirror/processes') self.add_tls(application='mirror') -- cgit From 6e28263d6531c9461a3a1d49a98bf8d33898c599 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Mon, 18 Jan 2021 16:48:47 +0300 Subject: Bumping year in copyright notice. --- NOTICE | 16 ++++++++-------- pkg/deb/debian/copyright | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/NOTICE b/NOTICE index 8d08b08b..ef97b24f 100644 --- a/NOTICE +++ b/NOTICE @@ -1,14 +1,14 @@ NGINX Unit. - Copyright 2017-2020 NGINX, Inc. - Copyright 2017-2020 Igor Sysoev - Copyright 2017-2020 Valentin V. Bartenev - Copyright 2017-2020 Max Romanov - Copyright 2017-2020 Andrei Zeliankou - Copyright 2017-2020 Andrei Belov - Copyright 2018-2020 Konstantin Pavlov - Copyright 2019-2020 Tiago Natel de Moura + Copyright 2017-2021 NGINX, Inc. + Copyright 2017-2021 Igor Sysoev + Copyright 2017-2021 Valentin V. Bartenev + Copyright 2017-2021 Max Romanov + Copyright 2017-2021 Andrei Zeliankou + Copyright 2017-2021 Andrei Belov + Copyright 2018-2021 Konstantin Pavlov + Copyright 2019-2021 Tiago Natel de Moura Copyright 2019-2020 Axel Duch Copyright 2018-2019 Alexander Borisov diff --git a/pkg/deb/debian/copyright b/pkg/deb/debian/copyright index 2acb87b9..2a2c5fd0 100644 --- a/pkg/deb/debian/copyright +++ b/pkg/deb/debian/copyright @@ -1,14 +1,14 @@ NGINX Unit. - Copyright 2017-2020 NGINX, Inc. - Copyright 2017-2020 Igor Sysoev - Copyright 2017-2020 Valentin V. Bartenev - Copyright 2017-2020 Max Romanov - Copyright 2017-2020 Andrei Zeliankou - Copyright 2017-2020 Andrei Belov - Copyright 2018-2020 Konstantin Pavlov - Copyright 2019-2020 Tiago Natel de Moura + Copyright 2017-2021 NGINX, Inc. + Copyright 2017-2021 Igor Sysoev + Copyright 2017-2021 Valentin V. Bartenev + Copyright 2017-2021 Max Romanov + Copyright 2017-2021 Andrei Zeliankou + Copyright 2017-2021 Andrei Belov + Copyright 2018-2021 Konstantin Pavlov + Copyright 2019-2021 Tiago Natel de Moura Copyright 2019-2020 Axel Duch Copyright 2018-2019 Alexander Borisov -- cgit From 9b76505bf7fe7954ebd66e23fc29dab2e6b02e47 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 25 Jan 2021 13:13:17 +0300 Subject: Router: fixing assertion in shortage of file descriptors. Each application in router process required fd for a request queue shared memory. When the number of file descripts close to the limit, and port sockets successfully opened, router needs to properly handle the errors. This patch closes port sockets before destroying port structure to avoid file descriptors leakage and assertion in debug build. --- src/nxt_router.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nxt_router.c b/src/nxt_router.c index 0416dea0..8d6e493d 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1586,6 +1586,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, ret = nxt_router_app_queue_init(task, port); if (nxt_slow_path(ret != NXT_OK)) { + nxt_port_write_close(port); + nxt_port_read_close(port); nxt_port_use(task, port, -1); return NXT_ERROR; } -- cgit From e4f7d1a29c59985f291876e2e76852007c2a390a Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Tue, 26 Jan 2021 13:06:30 +0300 Subject: Packages: added Fedora 33 support. --- pkg/rpm/Makefile | 4 +- pkg/rpm/Makefile.python39 | 57 ++++++++++++++++++++++ .../rpmbuild/SOURCES/unit.example-python39-config | 16 ++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 pkg/rpm/Makefile.python39 create mode 100644 pkg/rpm/rpmbuild/SOURCES/unit.example-python39-config diff --git a/pkg/rpm/Makefile b/pkg/rpm/Makefile index 285e0857..f0bd0cf9 100644 --- a/pkg/rpm/Makefile +++ b/pkg/rpm/Makefile @@ -128,7 +128,9 @@ include Makefile.php ifeq ($(shell test `rpm --eval '0%{?fedora} -lt 32'`; echo $$?),0) include Makefile.python27 endif -ifeq ($(shell test `rpm --eval '0%{?fedora} -ge 32'`; echo $$?),0) +ifeq ($(shell test `rpm --eval '0%{?fedora} -ge 33'`; echo $$?),0) +include Makefile.python39 +else ifeq ($(shell test `rpm --eval '0%{?fedora} -ge 32'`; echo $$?),0) include Makefile.python38 else ifeq ($(shell test `rpm --eval '0%{?fedora} -ge 29'`; echo $$?),0) include Makefile.python37 diff --git a/pkg/rpm/Makefile.python39 b/pkg/rpm/Makefile.python39 new file mode 100644 index 00000000..8e444e56 --- /dev/null +++ b/pkg/rpm/Makefile.python39 @@ -0,0 +1,57 @@ +MODULES+= python39 +MODULE_SUFFIX_python39= python3.9 + +MODULE_SUMMARY_python39= Python 3.9 module for NGINX Unit + +MODULE_VERSION_python39= $(VERSION) +MODULE_RELEASE_python39= 1 + +MODULE_CONFARGS_python39= python --config=python3.9-config +MODULE_MAKEARGS_python39= python3.9 +MODULE_INSTARGS_python39= python3.9-install + +MODULE_SOURCES_python39= unit.example-python-app \ + unit.example-python39-config + +ifneq (,$(findstring $(OSVER),opensuse-tumbleweed sles fedora amazonlinux2)) +BUILD_DEPENDS_python39= python3-devel +else +BUILD_DEPENDS_python39= python39-devel +endif + +BUILD_DEPENDS+= $(BUILD_DEPENDS_python39) + +define MODULE_PREINSTALL_python39 +%{__mkdir} -p %{buildroot}%{_datadir}/doc/unit-python39/examples/python-app +%{__install} -m 644 -p %{SOURCE100} \ + %{buildroot}%{_datadir}/doc/unit-python39/examples/python-app/wsgi.py +%{__install} -m 644 -p %{SOURCE101} \ + %{buildroot}%{_datadir}/doc/unit-python39/examples/unit.config +endef +export MODULE_PREINSTALL_python39 + +define MODULE_FILES_python39 +%{_libdir}/unit/modules/* +%{_libdir}/unit/debug-modules/* +endef +export MODULE_FILES_python39 + +define MODULE_POST_python39 +cat < Date: Wed, 27 Jan 2021 17:32:03 +0300 Subject: Router: fixing error handling in config request. The controller process awaits the response from the router for every configration change request. This patch adds error reporting for various error conditions which may happen because of file descriptors or memory shortage. Lack of a response lead to the controller awaiting the response, thus being unable to process other client reconfiguration requests that also became stuck. --- src/nxt_router.c | 61 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/src/nxt_router.c b/src/nxt_router.c index 8d6e493d..d9c722dd 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -699,26 +699,39 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) void *p; size_t size; nxt_int_t ret; + nxt_port_t *port; nxt_router_temp_conf_t *tmcf; + port = nxt_runtime_port_find(task->thread->runtime, + msg->port_msg.pid, + msg->port_msg.reply_port); + if (nxt_slow_path(port == NULL)) { + nxt_alert(task, "conf_data_handler: reply port not found"); + return; + } + + p = MAP_FAILED; + + /* + * Ancient compilers like gcc 4.8.5 on CentOS 7 wants 'size' to be + * initialized in 'cleanup' section. + */ + size = 0; + tmcf = nxt_router_temp_conf(task); if (nxt_slow_path(tmcf == NULL)) { - return; + goto fail; } if (nxt_slow_path(msg->fd[0] == -1)) { - nxt_alert(task, "conf_data_handler: invalid file shm fd"); - return; + nxt_alert(task, "conf_data_handler: invalid shm fd"); + goto fail; } if (nxt_buf_mem_used_size(&msg->buf->mem) != sizeof(size_t)) { nxt_alert(task, "conf_data_handler: unexpected buffer size (%d)", (int) nxt_buf_mem_used_size(&msg->buf->mem)); - - nxt_fd_close(msg->fd[0]); - msg->fd[0] = -1; - - return; + goto fail; } nxt_memcpy(&size, msg->buf->mem.pos, sizeof(size_t)); @@ -729,22 +742,14 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) msg->fd[0] = -1; if (nxt_slow_path(p == MAP_FAILED)) { - return; + goto fail; } nxt_debug(task, "conf_data_handler(%uz): %*s", size, size, p); tmcf->router_conf->router = nxt_router; tmcf->stream = msg->port_msg.stream; - tmcf->port = nxt_runtime_port_find(task->thread->runtime, - msg->port_msg.pid, - msg->port_msg.reply_port); - - if (nxt_slow_path(tmcf->port == NULL)) { - nxt_alert(task, "reply port not found"); - - goto fail; - } + tmcf->port = port; nxt_port_use(task, tmcf->port, 1); @@ -757,9 +762,27 @@ nxt_router_conf_data_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_router_conf_error(task, tmcf); } + goto cleanup; + fail: - nxt_mem_munmap(p, size); + nxt_port_socket_write(task, port, NXT_PORT_MSG_RPC_ERROR, -1, + msg->port_msg.stream, 0, NULL); + + if (tmcf != NULL) { + nxt_mp_destroy(tmcf->mem_pool); + } + +cleanup: + + if (p != MAP_FAILED) { + nxt_mem_munmap(p, size); + } + + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; + } } -- cgit From e4e444b82701de0c984a72eb9c2657f72d7171ae Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 28 Jan 2021 17:13:52 +0300 Subject: Router: fixing crash after WebSocket processing. After WebSocket processing, the application port was released with incorrect reason ("got request"), unnecessarily decrementing the active request counter. The assertion was triggered only on application removal; a test was added for this case. --- docs/changes.xml | 7 +++++++ src/nxt_router.c | 5 ++++- test/test_asgi_websockets.py | 24 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/changes.xml b/docs/changes.xml index b97cc823..64adbb46 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -97,6 +97,13 @@ a memory leak occurring in the router process when serving a file larger than + + +the router process could crash while removing or reconfiguring an application +that uses WebSocket. + + + diff --git a/src/nxt_router.c b/src/nxt_router.c index d9c722dd..03fe2a6c 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -3796,7 +3796,10 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_buf_chain_add(&b, nxt_http_buf_last(r)); req_rpc_data->rpc_cancel = 0; - req_rpc_data->apr_action = NXT_APR_GOT_RESPONSE; + + if (req_rpc_data->apr_action == NXT_APR_REQUEST_FAILED) { + req_rpc_data->apr_action = NXT_APR_GOT_RESPONSE; + } nxt_request_rpc_data_unlink(task, req_rpc_data); diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index 7c9ec555..6121fcc5 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -63,6 +63,9 @@ class TestASGIWebsockets(TestApplicationPython): key ), 'key' + # remove "mirror" application + self.load('websockets/subprotocol') + def test_asgi_websockets_subprotocol(self): self.load('websockets/subprotocol') @@ -92,6 +95,27 @@ class TestASGIWebsockets(TestApplicationPython): sock.close() + def test_asgi_websockets_mirror_app_change(self): + self.load('websockets/mirror') + + message = 'blah' + + _, sock, _ = self.ws.upgrade() + + self.ws.frame_write(sock, self.ws.OP_TEXT, message) + frame = self.ws.frame_read(sock) + + assert message == frame['data'].decode('utf-8'), 'mirror' + + self.load('websockets/subprotocol') + + self.ws.frame_write(sock, self.ws.OP_TEXT, message) + frame = self.ws.frame_read(sock) + + assert message == frame['data'].decode('utf-8'), 'mirror 2' + + sock.close() + def test_asgi_websockets_no_mask(self): self.load('websockets/mirror') -- cgit From 46a8c98a1aa6e0c4b86a72b8f8fe47878012afb1 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Thu, 28 Jan 2021 18:32:12 +0300 Subject: Removing unused mutex from nxt_process_t. --- src/nxt_process.h | 2 -- src/nxt_runtime.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/nxt_process.h b/src/nxt_process.h index 7afb8803..4f24b179 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -107,8 +107,6 @@ struct nxt_process_s { nxt_port_mmaps_t incoming; - nxt_thread_mutex_t cp_mutex; - uint32_t stream; nxt_mp_t *mem_pool; diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index d9d986da..8a86d38a 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -1387,7 +1387,6 @@ nxt_runtime_process_new(nxt_runtime_t *rt) nxt_queue_init(&process->ports); nxt_thread_mutex_create(&process->incoming.mutex); - nxt_thread_mutex_create(&process->cp_mutex); process->use_count = 1; @@ -1408,7 +1407,6 @@ nxt_runtime_process_release(nxt_runtime_t *rt, nxt_process_t *process) nxt_port_mmaps_destroy(&process->incoming, 1); nxt_thread_mutex_destroy(&process->incoming.mutex); - nxt_thread_mutex_destroy(&process->cp_mutex); /* processes from nxt_runtime_process_get() have no memory pool */ if (process->mem_pool != NULL) { -- cgit From 0ddbc00d7dd66ab83410bf02acf3b29fe946f518 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 28 Jan 2021 19:22:17 +0300 Subject: Fixed changelog style. --- docs/changes.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes.xml b/docs/changes.xml index 64adbb46..219ebfa2 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -65,7 +65,7 @@ Ruby apps with configured mutlithreading could crash on start under load. -Mount points were not unmounted when using "mount" namespace isolation; the +mount points were not unmounted when using "mount" namespace isolation; the bug had appeared in 1.21.0. -- cgit From df8ac184fd4c9f1e9cb10c2e3ca394e4bdae9e60 Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Wed, 13 Jan 2021 22:33:14 +0300 Subject: Docker: based docker images off official languages ones. Language versions are bumped to: - python 3.9 - ruby 2.7 - go 1.15 - node 15 - perl 5.32 Removed images: - python 2.7 due to EOL - full/latest due to incompatibility with the new scheme --- pkg/docker/Dockerfile.tmpl | 95 ------------------------------- pkg/docker/Makefile | 124 ++++++++++++++++++++++++++--------------- pkg/docker/template.Dockerfile | 73 ++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 141 deletions(-) delete mode 100644 pkg/docker/Dockerfile.tmpl create mode 100644 pkg/docker/template.Dockerfile diff --git a/pkg/docker/Dockerfile.tmpl b/pkg/docker/Dockerfile.tmpl deleted file mode 100644 index 8b0e35e4..00000000 --- a/pkg/docker/Dockerfile.tmpl +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION @@UNIT_VERSION@@ - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages=@@UNITPACKAGES@@ \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile index 455e9fb1..44fb5443 100644 --- a/pkg/docker/Makefile +++ b/pkg/docker/Makefile @@ -4,54 +4,95 @@ include ../../version include ../shasum.mak DEFAULT_VERSION := $(NXT_VERSION) -DEFAULT_RELEASE := 1 VERSION ?= $(DEFAULT_VERSION) -RELEASE ?= $(DEFAULT_RELEASE) -CODENAME := buster - -UNIT_VERSION = $(VERSION)-$(RELEASE)~$(CODENAME) - -MODULES = python2.7 python3.7 php7.3 go1.11-dev perl5.28 ruby2.5 \ - jsc11 full minimal - -MODULE_php7.3="unit=$${UNIT_VERSION} unit-php=$${UNIT_VERSION}" - -MODULE_python2.7="unit=$${UNIT_VERSION} unit-python2.7=$${UNIT_VERSION}" - -MODULE_python3.7="unit=$${UNIT_VERSION} unit-python3.7=$${UNIT_VERSION}" - -MODULE_go1.11-dev="unit=$${UNIT_VERSION} unit-go=$${UNIT_VERSION} gcc" - -MODULE_perl5.28="unit=$${UNIT_VERSION} unit-perl=$${UNIT_VERSION}" - -MODULE_ruby2.5="unit=$${UNIT_VERSION} unit-ruby=$${UNIT_VERSION}" - -MODULE_jsc11="unit=$${UNIT_VERSION} unit-jsc11=$${UNIT_VERSION}" - -MODULE_full="unit=$${UNIT_VERSION} unit-php=$${UNIT_VERSION} unit-python2.7=$${UNIT_VERSION} unit-python3.7=$${UNIT_VERSION} unit-perl=$${UNIT_VERSION} unit-ruby=$${UNIT_VERSION} unit-jsc11=$${UNIT_VERSION}" - -MODULE_minimal="unit=$${UNIT_VERSION}" EXPORT_DIR := $(VERSION) +MODULES ?= go jsc node perl php python ruby minimal + +VERSION_minimal ?= +CONTAINER_minimal ?= debian:buster-slim +CONFIGURE_minimal ?= +INSTALL_minimal ?= version +define COPY_minimal +endef + +VERSION_go ?= 1.15 +CONTAINER_go ?= golang:$(VERSION_go) +CONFIGURE_go ?= go --go-path=$$GOPATH +INSTALL_go ?= go-install-src libunit-install +define COPY_go +COPY --from=BUILDER /usr/lib/x86_64-linux-gnu/libunit.a /usr/lib/x86_64-linux-gnu/\n\$ +COPY --from=BUILDER /usr/include/nxt_* /usr/include/\n\$ +COPY --from=BUILDER /go/src/ /go/src/ +endef + +VERSION_jsc ?= 11 +CONTAINER_jsc ?= openjdk:$(VERSION_jsc)-jdk +CONFIGURE_jsc ?= java --jars=/usr/share/unit-jsc-common/ +INSTALL_jsc ?= java-shared-install java-install +COPY_jsc = COPY --from=BUILDER /usr/share/unit-jsc-common/ /usr/share/unit-jsc-common/ + +VERSION_node ?= 15 +CONTAINER_node ?= node:$(VERSION_node) +CONFIGURE_node ?= nodejs --node-gyp=/usr/local/lib/node_modules/npm/bin/node-gyp-bin/node-gyp +INSTALL_node ?= node node-install libunit-install +define COPY_node +COPY --from=BUILDER /usr/lib/x86_64-linux-gnu/libunit.a /usr/lib/x86_64-linux-gnu/\n\$ +COPY --from=BUILDER /usr/include/nxt_* /usr/include/\n\$ +COPY --from=BUILDER /usr/local/lib/node_modules/unit-http/ /usr/local/lib/node_modules/unit-http/ +endef + +VERSION_perl ?= 5.32 +CONTAINER_perl ?= perl:$(VERSION_perl) +CONFIGURE_perl ?= perl +INSTALL_perl ?= perl-install +COPY_perl = + +VERSION_php ?= 8.0 +CONTAINER_php ?= php:$(VERSION_php)-cli +CONFIGURE_php ?= php +INSTALL_php ?= php-install +COPY_php = RUN ldconfig + +VERSION_python ?= 3.9 +CONTAINER_python ?= python:$(VERSION_python) +CONFIGURE_python ?= python --config=/usr/local/bin/python3-config +INSTALL_python ?= python3-install +COPY_python = + +VERSION_ruby ?= 2.7 +CONTAINER_ruby ?= ruby:$(VERSION_ruby) +CONFIGURE_ruby ?= ruby +INSTALL_ruby ?= ruby-install +COPY_ruby = RUN gem install rack + default: @echo "valid targets: all build dockerfiles push tag export clean" -dockerfiles: $(addprefix Dockerfile., $(MODULES)) -build: refresh-base $(addprefix build-,$(MODULES)) -tag: $(addprefix tag-,$(MODULES)) -push: $(addprefix push-,$(MODULES)) latest -export: $(addsuffix .tar.gz,$(addprefix $(EXPORT_DIR)/nginx-unit-$(VERSION)-,$(MODULES))) $(addsuffix .tar.gz.sha512, $(addprefix $(EXPORT_DIR)/nginx-unit-$(VERSION)-,$(MODULES))) +MODVERSIONS = $(foreach module,$(MODULES),$(module)$(VERSION_$(module))) + +modname = $(shell echo $1 | /usr/bin/tr -d '.01234567890-') + +dockerfiles: $(addprefix Dockerfile., $(MODVERSIONS)) +build: $(addprefix build-,$(MODVERSIONS)) +tag: $(addprefix tag-,$(MODVERSIONS)) +push: $(addprefix push-,$(MODVERSIONS)) +export: $(addsuffix .tar.gz,$(addprefix $(EXPORT_DIR)/nginx-unit-$(VERSION)-,$(MODVERSIONS))) $(addsuffix .tar.gz.sha512, $(addprefix $(EXPORT_DIR)/nginx-unit-$(VERSION)-,$(MODVERSIONS))) Dockerfile.%: ../../version @echo "===> Building $@" - cat Dockerfile.tmpl | sed \ - -e 's,@@UNITPACKAGES@@,$(MODULE_$*),g' \ - -e 's,@@UNIT_VERSION@@,$(UNIT_VERSION),g' \ + cat template.Dockerfile | sed \ + -e 's,@@VERSION@@,$(VERSION),g' \ + -e 's,@@CONTAINER@@,$(CONTAINER_$(call modname, $*)),g' \ + -e 's,@@CONFIGURE@@,$(CONFIGURE_$(call modname, $*)),g' \ + -e 's,@@INSTALL@@,$(INSTALL_$(call modname, $*)),g' \ + -e 's,@@COPY@@,$(COPY_$(call modname, $*)),g' \ > $@ build-%: Dockerfile.% + docker pull $(CONTAINER_$(call modname, $*)) docker build --no-cache -t unit:$(VERSION)-$* -f Dockerfile.$* . tag-%: build-% @@ -62,15 +103,6 @@ push-%: tag-% docker push nginx/unit:$(VERSION)-$* docker push public.ecr.aws/nginx/unit:$(VERSION)-$* -latest: - docker tag nginx/unit:$(VERSION)-full nginx/unit:latest - docker tag nginx/unit:$(VERSION)-full public.ecr.aws/nginx/unit:latest - docker push nginx/unit:latest - docker push public.ecr.aws/nginx/unit:latest - -refresh-base: - docker pull $(shell head -n 1 Dockerfile.tmpl | cut -d' ' -f 2) - $(EXPORT_DIR): mkdir -p $@ @@ -80,10 +112,10 @@ $(EXPORT_DIR)/nginx-unit-$(VERSION)-%.tar.gz: $(EXPORT_DIR) tag-% $(EXPORT_DIR)/nginx-unit-$(VERSION)-%.tar.gz.sha512: $(EXPORT_DIR)/nginx-unit-$(VERSION)-%.tar.gz $(SHA512SUM) $< | sed 's,$(EXPORT_DIR)/,,' > $@ -all: $(addprefix Dockerfile., $(MODULES)) +all: $(addprefix Dockerfile., $(MODVERSIONS)) clean: - rm -f $(addprefix Dockerfile., $(MODULES)) + rm -f $(addprefix Dockerfile., $(MODVERSIONS)) rm -rf $(EXPORT_DIR) -.PHONY: default all build dockerfiles latest push tag export clean refresh-base +.PHONY: default build dockerfiles push tag export clean diff --git a/pkg/docker/template.Dockerfile b/pkg/docker/template.Dockerfile new file mode 100644 index 00000000..d96d7982 --- /dev/null +++ b/pkg/docker/template.Dockerfile @@ -0,0 +1,73 @@ +FROM @@CONTAINER@@ as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up @@VERSION@@ \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure @@CONFIGURE@@ \ + && make -j $NCPU @@INSTALL@@ \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure @@CONFIGURE@@ \ + && make -j $NCPU @@INSTALL@@ \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM @@CONTAINER@@ +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt +@@COPY@@ +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] -- cgit From 510bdf15bec094d49be8aba63e56963aed5746cf Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Mon, 1 Feb 2021 18:05:12 +0300 Subject: Regenerated Dockerfiles. --- pkg/docker/Dockerfile.full | 95 ------------------------- pkg/docker/Dockerfile.go1.11-dev | 95 ------------------------- pkg/docker/Dockerfile.go1.15 | 75 ++++++++++++++++++++ pkg/docker/Dockerfile.jsc11 | 146 +++++++++++++++++---------------------- pkg/docker/Dockerfile.minimal | 146 +++++++++++++++++---------------------- pkg/docker/Dockerfile.node15 | 75 ++++++++++++++++++++ pkg/docker/Dockerfile.perl5.28 | 95 ------------------------- pkg/docker/Dockerfile.perl5.32 | 73 ++++++++++++++++++++ pkg/docker/Dockerfile.php7.3 | 95 ------------------------- pkg/docker/Dockerfile.php8.0 | 73 ++++++++++++++++++++ pkg/docker/Dockerfile.python2.7 | 95 ------------------------- pkg/docker/Dockerfile.python3.7 | 95 ------------------------- pkg/docker/Dockerfile.python3.9 | 73 ++++++++++++++++++++ pkg/docker/Dockerfile.ruby2.5 | 95 ------------------------- pkg/docker/Dockerfile.ruby2.7 | 73 ++++++++++++++++++++ 15 files changed, 566 insertions(+), 833 deletions(-) delete mode 100644 pkg/docker/Dockerfile.full delete mode 100644 pkg/docker/Dockerfile.go1.11-dev create mode 100644 pkg/docker/Dockerfile.go1.15 create mode 100644 pkg/docker/Dockerfile.node15 delete mode 100644 pkg/docker/Dockerfile.perl5.28 create mode 100644 pkg/docker/Dockerfile.perl5.32 delete mode 100644 pkg/docker/Dockerfile.php7.3 create mode 100644 pkg/docker/Dockerfile.php8.0 delete mode 100644 pkg/docker/Dockerfile.python2.7 delete mode 100644 pkg/docker/Dockerfile.python3.7 create mode 100644 pkg/docker/Dockerfile.python3.9 delete mode 100644 pkg/docker/Dockerfile.ruby2.5 create mode 100644 pkg/docker/Dockerfile.ruby2.7 diff --git a/pkg/docker/Dockerfile.full b/pkg/docker/Dockerfile.full deleted file mode 100644 index a5f3e13f..00000000 --- a/pkg/docker/Dockerfile.full +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-php=${UNIT_VERSION} unit-python2.7=${UNIT_VERSION} unit-python3.7=${UNIT_VERSION} unit-perl=${UNIT_VERSION} unit-ruby=${UNIT_VERSION} unit-jsc11=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.go1.11-dev b/pkg/docker/Dockerfile.go1.11-dev deleted file mode 100644 index cb1c1d85..00000000 --- a/pkg/docker/Dockerfile.go1.11-dev +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-go=${UNIT_VERSION} gcc" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.go1.15 b/pkg/docker/Dockerfile.go1.15 new file mode 100644 index 00000000..9f9b8faa --- /dev/null +++ b/pkg/docker/Dockerfile.go1.15 @@ -0,0 +1,75 @@ +FROM golang:1.15 as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure go --go-path=$GOPATH \ + && make -j $NCPU go-install-src libunit-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure go --go-path=$GOPATH \ + && make -j $NCPU go-install-src libunit-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM golang:1.15 +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt +COPY --from=BUILDER /usr/lib/x86_64-linux-gnu/libunit.a /usr/lib/x86_64-linux-gnu/ +COPY --from=BUILDER /usr/include/nxt_* /usr/include/ +COPY --from=BUILDER /go/src/ /go/src/ +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.jsc11 b/pkg/docker/Dockerfile.jsc11 index f3d1d22d..ce0d73f3 100644 --- a/pkg/docker/Dockerfile.jsc11 +++ b/pkg/docker/Dockerfile.jsc11 @@ -1,95 +1,73 @@ -FROM debian:buster-slim +FROM openjdk:11-jdk as BUILDER LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.21.0-1~buster +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure java --jars=/usr/share/unit-jsc-common/ \ + && make -j $NCPU java-shared-install java-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure java --jars=/usr/share/unit-jsc-common/ \ + && make -j $NCPU java-shared-install java-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt +FROM openjdk:11-jdk +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt +COPY --from=BUILDER /usr/share/unit-jsc-common/ /usr/share/unit-jsc-common/ RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-jsc11=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log STOPSIGNAL SIGTERM -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index da9c8712..4030816a 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -1,95 +1,73 @@ -FROM debian:buster-slim +FROM debian:buster-slim as BUILDER LABEL maintainer="NGINX Docker Maintainers " -ENV UNIT_VERSION 1.21.0-1~buster +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure \ + && make -j $NCPU version \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure \ + && make -j $NCPU version \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi +FROM debian:buster-slim +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log STOPSIGNAL SIGTERM -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.node15 b/pkg/docker/Dockerfile.node15 new file mode 100644 index 00000000..0f05e2be --- /dev/null +++ b/pkg/docker/Dockerfile.node15 @@ -0,0 +1,75 @@ +FROM node:15 as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure nodejs --node-gyp=/usr/local/lib/node_modules/npm/bin/node-gyp-bin/node-gyp \ + && make -j $NCPU node node-install libunit-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure nodejs --node-gyp=/usr/local/lib/node_modules/npm/bin/node-gyp-bin/node-gyp \ + && make -j $NCPU node node-install libunit-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM node:15 +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt +COPY --from=BUILDER /usr/lib/x86_64-linux-gnu/libunit.a /usr/lib/x86_64-linux-gnu/ +COPY --from=BUILDER /usr/include/nxt_* /usr/include/ +COPY --from=BUILDER /usr/local/lib/node_modules/unit-http/ /usr/local/lib/node_modules/unit-http/ +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.perl5.28 b/pkg/docker/Dockerfile.perl5.28 deleted file mode 100644 index 976fa7b0..00000000 --- a/pkg/docker/Dockerfile.perl5.28 +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-perl=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.perl5.32 b/pkg/docker/Dockerfile.perl5.32 new file mode 100644 index 00000000..0b349cd4 --- /dev/null +++ b/pkg/docker/Dockerfile.perl5.32 @@ -0,0 +1,73 @@ +FROM perl:5.32 as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure perl \ + && make -j $NCPU perl-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure perl \ + && make -j $NCPU perl-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM perl:5.32 +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt + +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.php7.3 b/pkg/docker/Dockerfile.php7.3 deleted file mode 100644 index f1178551..00000000 --- a/pkg/docker/Dockerfile.php7.3 +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-php=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.php8.0 b/pkg/docker/Dockerfile.php8.0 new file mode 100644 index 00000000..b8c9c046 --- /dev/null +++ b/pkg/docker/Dockerfile.php8.0 @@ -0,0 +1,73 @@ +FROM php:8.0-cli as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure php \ + && make -j $NCPU php-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure php \ + && make -j $NCPU php-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM php:8.0-cli +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt +RUN ldconfig +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.python2.7 b/pkg/docker/Dockerfile.python2.7 deleted file mode 100644 index 95241af6..00000000 --- a/pkg/docker/Dockerfile.python2.7 +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-python2.7=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.python3.7 b/pkg/docker/Dockerfile.python3.7 deleted file mode 100644 index 52a5a257..00000000 --- a/pkg/docker/Dockerfile.python3.7 +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-python3.7=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.python3.9 b/pkg/docker/Dockerfile.python3.9 new file mode 100644 index 00000000..a0080fbb --- /dev/null +++ b/pkg/docker/Dockerfile.python3.9 @@ -0,0 +1,73 @@ +FROM python:3.9 as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure python --config=/usr/local/bin/python3-config \ + && make -j $NCPU python3-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure python --config=/usr/local/bin/python3-config \ + && make -j $NCPU python3-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM python:3.9 +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt + +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.ruby2.5 b/pkg/docker/Dockerfile.ruby2.5 deleted file mode 100644 index 342ccc9a..00000000 --- a/pkg/docker/Dockerfile.ruby2.5 +++ /dev/null @@ -1,95 +0,0 @@ -FROM debian:buster-slim - -LABEL maintainer="NGINX Docker Maintainers " - -ENV UNIT_VERSION 1.21.0-1~buster - -RUN set -x \ - && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 apt-transport-https ca-certificates \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - ha.pool.sks-keyservers.net \ - hkp://keyserver.ubuntu.com:80 \ - hkp://p80.pool.sks-keyservers.net:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# work-around debian bug 863199 - && mkdir -p /usr/share/man/man1 \ - && dpkgArch="$(dpkg --print-architecture)" \ - && unitPackages="unit=${UNIT_VERSION} unit-ruby=${UNIT_VERSION}" \ - && case "$dpkgArch" in \ - amd64|i386) \ -# arches officialy built by upstream - echo "deb https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - && apt-get update \ - ;; \ - *) \ -# we're on an architecture upstream doesn't officially build for -# let's build binaries from the published source packages - echo "deb-src https://packages.nginx.org/unit/debian/ buster unit" >> /etc/apt/sources.list.d/unit.list \ - \ -# new directory for storing sources and .deb files - && tempDir="$(mktemp -d)" \ - && chmod 777 "$tempDir" \ -# (777 to ensure APT's "_apt" user can access it too) - \ -# save list of currently-installed packages so build dependencies can be cleanly removed later - && savedAptMark="$(apt-mark showmanual)" \ - \ -# build .deb files from upstream's source packages (which are verified by apt-get) - && apt-get update \ - && apt-get build-dep -y $unitPackages \ - && ( \ - cd "$tempDir" \ - && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \ - apt-get source --compile $unitPackages \ - ) \ -# we don't remove APT lists here because they get re-downloaded and removed later - \ -# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies -# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies) - && apt-mark showmanual | xargs apt-mark auto > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ - \ -# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be) - && ls -lAFh "$tempDir" \ - && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \ - && grep '^Package: ' "$tempDir/Packages" \ - && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \ -# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes") -# Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) -# ... -# E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied) - && apt-get -o Acquire::GzipIndexes=false update \ - ;; \ - esac \ - \ - && apt-get install --no-install-recommends --no-install-suggests -y \ - $unitPackages \ - curl \ - && apt-get remove --purge --auto-remove -y apt-transport-https && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/unit.list \ - \ -# if we have leftovers from building, let's purge them (including extra, unnecessary build deps) - && if [ -n "$tempDir" ]; then \ - apt-get purge -y --auto-remove \ - && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \ - fi - -# forward log to docker log collector -RUN ln -sf /dev/stdout /var/log/unit.log - -STOPSIGNAL SIGTERM - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN mkdir /docker-entrypoint.d/ -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] - -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] diff --git a/pkg/docker/Dockerfile.ruby2.7 b/pkg/docker/Dockerfile.ruby2.7 new file mode 100644 index 00000000..ad3eec68 --- /dev/null +++ b/pkg/docker/Dockerfile.ruby2.7 @@ -0,0 +1,73 @@ +FROM ruby:2.7 as BUILDER + +LABEL maintainer="NGINX Docker Maintainers " + +RUN set -ex \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates mercurial build-essential libssl-dev libpcre2-dev \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && hg clone https://hg.nginx.org/unit \ + && cd unit \ + && hg up 1.21.0 \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS="--prefix=/usr \ + --state=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --pid=/var/run/unit.pid \ + --log=/var/log/unit.log \ + --tmp=/var/tmp \ + --user=unit \ + --group=unit \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modules=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/unitd /usr/sbin/unitd \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/debug-modules --debug \ + && ./configure ruby \ + && make -j $NCPU ruby-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --modules=/usr/lib/unit/modules \ + && ./configure ruby \ + && make -j $NCPU ruby-install \ + && ldd /usr/sbin/unitd | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq > /requirements.apt + +FROM ruby:2.7 +COPY docker-entrypoint.sh /usr/local/bin/ +COPY --from=BUILDER /usr/sbin/unitd /usr/sbin/unitd +COPY --from=BUILDER /usr/sbin/unitd-debug /usr/sbin/unitd-debug +COPY --from=BUILDER /usr/lib/unit/ /usr/lib/unit/ +COPY --from=BUILDER /requirements.apt /requirements.apt +RUN gem install rack +RUN set -x \ + && mkdir -p /var/lib/unit/ \ + && mkdir /docker-entrypoint.d/ \ + && addgroup --system unit \ + && adduser \ + --system \ + --disabled-login \ + --ingroup unit \ + --no-create-home \ + --home /nonexistent \ + --gecos "unit user" \ + --shell /bin/false \ + unit \ + && apt update \ + && apt --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stdout /var/log/unit.log + +STOPSIGNAL SIGTERM + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock"] -- cgit From 93ac087e9684c63f82df36f847bf9239e2eb185e Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Mon, 1 Feb 2021 18:55:49 +0300 Subject: Fixed building by GCC 10 with -flto and -O2. This closes #467 issue on GitHub. --- src/nxt_http_route.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 9aaa708e..28545fc9 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -813,6 +813,12 @@ nxt_http_route_ruleset_create(nxt_task_t *task, nxt_mp_t *mp, next = 0; + /* + * A workaround for GCC 10 with -flto -O2 flags that warns about "name" + * may be uninitialized in nxt_http_route_rule_name_create(). + */ + nxt_str_null(&name); + for (i = 0; i < n; i++) { rule_cv = nxt_conf_next_object_member(ruleset_cv, &name, &next); -- cgit From d4d704bc7f37f457a50b1226165d827a3c2b25e1 Mon Sep 17 00:00:00 2001 From: Konstantin Pavlov Date: Tue, 2 Feb 2021 16:46:10 +0300 Subject: Docker: reverted 44154b830401. Automatic synchronisation is now set up, so manual pushes to AWS ECR are discouraged. --- pkg/docker/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/docker/Makefile b/pkg/docker/Makefile index 44fb5443..00625526 100644 --- a/pkg/docker/Makefile +++ b/pkg/docker/Makefile @@ -97,11 +97,9 @@ build-%: Dockerfile.% tag-%: build-% docker tag unit:$(VERSION)-$* nginx/unit:$(VERSION)-$* - docker tag unit:$(VERSION)-$* public.ecr.aws/nginx/unit:$(VERSION)-$* push-%: tag-% docker push nginx/unit:$(VERSION)-$* - docker push public.ecr.aws/nginx/unit:$(VERSION)-$* $(EXPORT_DIR): mkdir -p $@ -- cgit From 8c88537e6ee0c0a2ae1c323b8cce09522240471b Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 3 Feb 2021 23:23:06 +0300 Subject: Using shared memory to pass configuration to main process. This patch is required to remove fragmented messages functionality. --- src/nxt_controller.c | 37 +++++++++++++++++++++++---- src/nxt_main_process.c | 69 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/nxt_controller.c b/src/nxt_controller.c index 9a34a877..772d10c8 100644 --- a/src/nxt_controller.c +++ b/src/nxt_controller.c @@ -1686,7 +1686,10 @@ nxt_controller_conf_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, static void nxt_controller_conf_store(nxt_task_t *task, nxt_conf_value_t *conf) { + void *mem; + u_char *end; size_t size; + nxt_fd_t fd; nxt_buf_t *b; nxt_port_t *main_port; nxt_runtime_t *rt; @@ -1697,14 +1700,38 @@ nxt_controller_conf_store(nxt_task_t *task, nxt_conf_value_t *conf) size = nxt_conf_json_length(conf, NULL); - b = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, size); + fd = nxt_shm_open(task, size); + if (nxt_slow_path(fd == -1)) { + return; + } + + mem = nxt_mem_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (nxt_slow_path(mem == MAP_FAILED)) { + goto fail; + } + + end = nxt_conf_json_print(mem, conf, NULL); + + nxt_mem_munmap(mem, size); - if (nxt_fast_path(b != NULL)) { - b->mem.free = nxt_conf_json_print(b->mem.free, conf, NULL); + size = end - (u_char *) mem; - (void) nxt_port_socket_write(task, main_port, NXT_PORT_MSG_CONF_STORE, - -1, 0, -1, b); + b = nxt_buf_mem_alloc(task->thread->engine->mem_pool, sizeof(size_t), 0); + if (nxt_slow_path(b == NULL)) { + goto fail; } + + b->mem.free = nxt_cpymem(b->mem.pos, &size, sizeof(size_t)); + + (void) nxt_port_socket_write(task, main_port, + NXT_PORT_MSG_CONF_STORE | NXT_PORT_MSG_CLOSE_FD, + fd, 0, -1, b); + + return; + +fail: + + nxt_fd_close(fd); } diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 2916f0ab..9a78f9da 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1408,12 +1408,45 @@ nxt_app_lang_compare(const void *v1, const void *v2) static void nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) { - ssize_t n, size, offset; - nxt_buf_t *b; + void *p; + size_t size; + ssize_t n; nxt_int_t ret; nxt_file_t file; nxt_runtime_t *rt; + p = MAP_FAILED; + + /* + * Ancient compilers like gcc 4.8.5 on CentOS 7 wants 'size' to be + * initialized in 'cleanup' section. + */ + size = 0; + + if (nxt_slow_path(msg->fd[0] == -1)) { + nxt_alert(task, "conf_store_handler: invalid shm fd"); + goto error; + } + + if (nxt_buf_mem_used_size(&msg->buf->mem) != sizeof(size_t)) { + nxt_alert(task, "conf_store_handler: unexpected buffer size (%d)", + (int) nxt_buf_mem_used_size(&msg->buf->mem)); + goto error; + } + + nxt_memcpy(&size, msg->buf->mem.pos, sizeof(size_t)); + + p = nxt_mem_mmap(NULL, size, PROT_READ, MAP_SHARED, msg->fd[0], 0); + + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; + + if (nxt_slow_path(p == MAP_FAILED)) { + goto error; + } + + nxt_debug(task, "conf_store_handler(%uz): %*s", size, size, p); + nxt_memzero(&file, sizeof(nxt_file_t)); rt = task->thread->runtime; @@ -1427,33 +1460,35 @@ nxt_main_port_conf_store_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) goto error; } - offset = 0; - - for (b = msg->buf; b != NULL; b = b->next) { - size = nxt_buf_mem_used_size(&b->mem); - - n = nxt_file_write(&file, b->mem.pos, size, offset); + n = nxt_file_write(&file, p, size, 0); - if (nxt_slow_path(n != size)) { - nxt_file_close(task, &file); - (void) nxt_file_delete(file.name); - goto error; - } + nxt_file_close(task, &file); - offset += n; + if (nxt_slow_path(n != (ssize_t) size)) { + (void) nxt_file_delete(file.name); + goto error; } - nxt_file_close(task, &file); - ret = nxt_file_rename(file.name, (nxt_file_name_t *) rt->conf); if (nxt_fast_path(ret == NXT_OK)) { - return; + goto cleanup; } error: nxt_alert(task, "failed to store current configuration"); + +cleanup: + + if (p != MAP_FAILED) { + nxt_mem_munmap(p, size); + } + + if (msg->fd[0] != -1) { + nxt_fd_close(msg->fd[0]); + msg->fd[0] = -1; + } } -- cgit From b1685dbc769a1b62eedc3249697c30d8770fc8c3 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 3 Feb 2021 23:23:17 +0300 Subject: Fixing possible NULL dereference. For listen socket request reply port can be NULL if Router crashes immediately after issuing the request. Found by Coverity (CID 366310). --- src/nxt_main_process.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index 9a78f9da..f20f2c2c 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1001,21 +1001,22 @@ nxt_main_port_socket_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_listening_socket_t ls; u_char message[2048]; + port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, + msg->port_msg.reply_port); + if (nxt_slow_path(port == NULL)) { + return; + } + b = msg->buf; sa = (nxt_sockaddr_t *) b->mem.pos; /* TODO check b size and make plain */ - out = NULL; - ls.socket = -1; ls.error = NXT_SOCKET_ERROR_SYSTEM; ls.start = message; ls.end = message + sizeof(message); - port = nxt_runtime_port_find(task->thread->runtime, msg->port_msg.pid, - msg->port_msg.reply_port); - nxt_debug(task, "listening socket \"%*s\"", (size_t) sa->length, nxt_sockaddr_start(sa)); @@ -1025,6 +1026,8 @@ nxt_main_port_socket_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_debug(task, "socket(\"%*s\"): %d", (size_t) sa->length, nxt_sockaddr_start(sa), ls.socket); + out = NULL; + type = NXT_PORT_MSG_RPC_READY_LAST | NXT_PORT_MSG_CLOSE_FD; } else { @@ -1034,13 +1037,11 @@ nxt_main_port_socket_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) out = nxt_buf_mem_ts_alloc(task, task->thread->engine->mem_pool, size + 1); - if (nxt_slow_path(out == NULL)) { - return; - } - - *out->mem.free++ = (uint8_t) ls.error; + if (nxt_fast_path(out != NULL)) { + *out->mem.free++ = (uint8_t) ls.error; - out->mem.free = nxt_cpymem(out->mem.free, ls.start, size); + out->mem.free = nxt_cpymem(out->mem.free, ls.start, size); + } type = NXT_PORT_MSG_RPC_ERROR; } -- cgit From 75a5dcfc4ec4f92a196c0cf3a187081a238a6b1d Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Wed, 3 Feb 2021 23:23:28 +0300 Subject: Fixing shared app queue unmap size. Shared app queue takes more memory than port memory. To unmap all memory pages correct size need to be specified for munmap() call. Otherwise 4 Mb memory leaked on each configured application removal. The issue was introduced in 1d84b9e4b459. --- docs/changes.xml | 7 +++++++ src/nxt_port.c | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changes.xml b/docs/changes.xml index 219ebfa2..d12e73c6 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -104,6 +104,13 @@ that uses WebSocket. + + +a memory leak occurring in the router process when removing or reconfiguring +an application; the bug had appeared in 1.19.0. + + + diff --git a/src/nxt_port.c b/src/nxt_port.c index dbcdec11..d4e46564 100644 --- a/src/nxt_port.c +++ b/src/nxt_port.c @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -84,6 +85,8 @@ nxt_port_new(nxt_task_t *task, nxt_port_id_t id, nxt_pid_t pid, void nxt_port_close(nxt_task_t *task, nxt_port_t *port) { + size_t size; + nxt_debug(task, "port %p %d:%d close, type %d", port, port->pid, port->id, port->type); @@ -109,7 +112,10 @@ nxt_port_close(nxt_task_t *task, nxt_port_t *port) } if (port->queue != NULL) { - nxt_mem_munmap(port->queue, sizeof(nxt_port_queue_t)); + size = (port->id == (nxt_port_id_t) -1) ? sizeof(nxt_app_queue_t) + : sizeof(nxt_port_queue_t); + nxt_mem_munmap(port->queue, size); + port->queue = NULL; } } -- cgit From f2d96335664f561cab4ff2ff02f5cd3333af3b3a Mon Sep 17 00:00:00 2001 From: Artem Konev Date: Thu, 4 Feb 2021 14:34:42 +0000 Subject: Updated phrasing and corrected errors in docs/changes.xml. --- docs/changes.xml | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index d12e73c6..04507e8e 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -33,14 +33,15 @@ NGINX Unit updated to 1.22.0. -ability to specify multiple directories in the "path" option of Python apps. +support for specifying multiple directories in the "path" option of Python +apps. -ServerRequest and ServerResponse objects of Node.js module are now compliant -to Stream API. +the ServerRequest and ServerResponse objects of Node.js module are now +compliant with Stream API. @@ -52,55 +53,55 @@ invalid HTTP responses were generated for some unusual status codes. -PHP_AUTH_USER, PHP_AUTH_PW, and PHP_AUTH_DIGEST server variables were missing -in PHP module. +the PHP_AUTH_USER, PHP_AUTH_PW, and PHP_AUTH_DIGEST server variables were +missing in the PHP module. -Ruby apps with configured mutlithreading could crash on start under load. +Ruby apps with multithreading configured could crash on start under load. -mount points were not unmounted when using "mount" namespace isolation; the -bug had appeared in 1.21.0. +mount points weren't unmounted when the "mount" namespace isolation was used; +the bug had appeared in 1.21.0. -the router process could crash with multithreaded applications under high load. +the router process could crash with multithreaded apps under high load. -applications could stop processing new requests under high load; the bug had +apps could stop processing new requests under high load; the bug had appeared in 1.19.0. -application processes could terminate unexpectedly under high load; the bug -had appeared in 1.19.0. +app processes could terminate unexpectedly under high load; the bug had +appeared in 1.19.0. -a memory leak occurring in the router process when serving a file larger than +a memory leak occurred in the router process when serving files larger than 128K; the bug had appeared in 1.13.0. -the router process could crash while removing or reconfiguring an application -that uses WebSocket. +the router process could crash while removing or reconfiguring an app that used +WebSocket. -- cgit From 42725137f7a19991680c78a0b2d69bcbf1f9ab63 Mon Sep 17 00:00:00 2001 From: Andrei Zeliankou Date: Thu, 4 Feb 2021 15:09:54 +0000 Subject: Tests: added tests for "path" option in Python application. --- test/python/path/wsgi.py | 8 ++++++++ test/test_python_application.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 test/python/path/wsgi.py diff --git a/test/python/path/wsgi.py b/test/python/path/wsgi.py new file mode 100644 index 00000000..2807f6ef --- /dev/null +++ b/test/python/path/wsgi.py @@ -0,0 +1,8 @@ +import os +import sys + +def application(environ, start_response): + body = os.pathsep.join(sys.path).encode() + + start_response('200', [('Content-Length', str(len(body)))]) + return [body] diff --git a/test/test_python_application.py b/test/test_python_application.py index 5ad0901d..709df3ff 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -1,4 +1,5 @@ import grp +import os import pwd import re import time @@ -790,6 +791,42 @@ last line: 987654321 assert self.get()['status'] not in [200, 204], 'callable response inv' + def test_python_application_path(self): + self.load('path') + + def set_path(path): + assert 'success' in self.conf(path, 'applications/path/path') + + def get_path(): + return self.get()['body'].split(os.pathsep) + + default_path = self.conf_get('/config/applications/path/path') + assert 'success' in self.conf( + {"PYTHONPATH": default_path}, + '/config/applications/path/environment', + ) + + self.conf_delete('/config/applications/path/path') + sys_path = get_path() + + set_path('"/blah"') + assert ['/blah', *sys_path] == get_path(), 'check path' + + set_path('"/new"') + assert ['/new', *sys_path] == get_path(), 'check path update' + + set_path('["/blah1", "/blah2"]') + assert ['/blah1', '/blah2', *sys_path] == get_path(), 'check path array' + + def test_python_application_path_invalid(self): + self.load('path') + + def check_path(path): + assert 'error' in self.conf(path, 'applications/path/path') + + check_path('{}') + check_path('["/blah", []]') + def test_python_application_threads(self): self.load('threads', threads=4) -- cgit From bde6a2afd4c3230f070e2e9158b6a96d1ce16b89 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 4 Feb 2021 18:22:33 +0300 Subject: Reordered changes for 1.22.0 by significance (subjective). --- docs/changes.xml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/changes.xml b/docs/changes.xml index 04507e8e..35eaec6c 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -33,68 +33,68 @@ NGINX Unit updated to 1.22.0. -support for specifying multiple directories in the "path" option of Python -apps. +the ServerRequest and ServerResponse objects of Node.js module are now +compliant with Stream API. -the ServerRequest and ServerResponse objects of Node.js module are now -compliant with Stream API. +support for specifying multiple directories in the "path" option of Python +apps. -invalid HTTP responses were generated for some unusual status codes. +a memory leak occurred in the router process when serving files larger than +128K; the bug had appeared in 1.13.0. -the PHP_AUTH_USER, PHP_AUTH_PW, and PHP_AUTH_DIGEST server variables were -missing in the PHP module. +apps could stop processing new requests under high load; the bug had +appeared in 1.19.0. -Ruby apps with multithreading configured could crash on start under load. +app processes could terminate unexpectedly under high load; the bug had +appeared in 1.19.0. -mount points weren't unmounted when the "mount" namespace isolation was used; -the bug had appeared in 1.21.0. +invalid HTTP responses were generated for some unusual status codes. -the router process could crash with multithreaded apps under high load. +the PHP_AUTH_USER, PHP_AUTH_PW, and PHP_AUTH_DIGEST server variables were +missing in the PHP module. -apps could stop processing new requests under high load; the bug had -appeared in 1.19.0. +the router process could crash with multithreaded apps under high load. -app processes could terminate unexpectedly under high load; the bug had -appeared in 1.19.0. +Ruby apps with multithreading configured could crash on start under load. -a memory leak occurred in the router process when serving files larger than -128K; the bug had appeared in 1.13.0. +mount points weren't unmounted when the "mount" namespace isolation was used; +the bug had appeared in 1.21.0. -- cgit From 56626b9232857d49deeca15d7dc8eeaeaffbdd6d Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 4 Feb 2021 18:22:33 +0300 Subject: Added version 1.22.0 CHANGES. --- CHANGES | 39 +++++++++++++++++++++++++++++++++++++++ docs/changes.xml | 4 ++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index b026fe5e..0f558c6f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,43 @@ +Changes with Unit 1.22.0 04 Feb 2021 + + *) Feature: the ServerRequest and ServerResponse objects of Node.js + module are now compliant with Stream API. + + *) Feature: support for specifying multiple directories in the "path" + option of Python apps. + + *) Bugfix: a memory leak occurred in the router process when serving + files larger than 128K; the bug had appeared in 1.13.0. + + *) Bugfix: apps could stop processing new requests under high load; the + bug had appeared in 1.19.0. + + *) Bugfix: app processes could terminate unexpectedly under high load; + the bug had appeared in 1.19.0. + + *) Bugfix: invalid HTTP responses were generated for some unusual status + codes. + + *) Bugfix: the PHP_AUTH_USER, PHP_AUTH_PW, and PHP_AUTH_DIGEST server + variables were missing in the PHP module. + + *) Bugfix: the router process could crash with multithreaded apps under + high load. + + *) Bugfix: Ruby apps with multithreading configured could crash on start + under load. + + *) Bugfix: mount points weren't unmounted when the "mount" namespace + isolation was used; the bug had appeared in 1.21.0. + + *) Bugfix: the router process could crash while removing or + reconfiguring an app that used WebSocket. + + *) Bugfix: a memory leak occurring in the router process when removing + or reconfiguring an application; the bug had appeared in 1.19.0. + + Changes with Unit 1.21.0 19 Nov 2020 *) Change: procfs is mounted by default for all languages when "rootfs" diff --git a/docs/changes.xml b/docs/changes.xml index 35eaec6c..378320f4 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -15,7 +15,7 @@ unit-jsc-common unit-jsc8 unit-jsc10 unit-jsc11 unit-jsc13 unit-jsc14 unit-jsc15" ver="1.22.0" rev="1" - date="" time="" + date="2021-02-04" time="18:00:00 +0300" packager="Andrei Belov <defan@nginx.com>"> @@ -28,7 +28,7 @@ NGINX Unit updated to 1.22.0. -- cgit From 98228f8ac878799e8265658d990bbff94675856b Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 4 Feb 2021 18:22:33 +0300 Subject: Generated Dockerfiles for Unit 1.22.0. --- pkg/docker/Dockerfile.go1.15 | 2 +- pkg/docker/Dockerfile.jsc11 | 2 +- pkg/docker/Dockerfile.minimal | 2 +- pkg/docker/Dockerfile.node15 | 2 +- pkg/docker/Dockerfile.perl5.32 | 2 +- pkg/docker/Dockerfile.php8.0 | 2 +- pkg/docker/Dockerfile.python3.9 | 2 +- pkg/docker/Dockerfile.ruby2.7 | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/docker/Dockerfile.go1.15 b/pkg/docker/Dockerfile.go1.15 index 9f9b8faa..270dc428 100644 --- a/pkg/docker/Dockerfile.go1.15 +++ b/pkg/docker/Dockerfile.go1.15 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.jsc11 b/pkg/docker/Dockerfile.jsc11 index ce0d73f3..cde7c590 100644 --- a/pkg/docker/Dockerfile.jsc11 +++ b/pkg/docker/Dockerfile.jsc11 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 4030816a..91fd79d0 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.node15 b/pkg/docker/Dockerfile.node15 index 0f05e2be..59b12012 100644 --- a/pkg/docker/Dockerfile.node15 +++ b/pkg/docker/Dockerfile.node15 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.perl5.32 b/pkg/docker/Dockerfile.perl5.32 index 0b349cd4..589eb989 100644 --- a/pkg/docker/Dockerfile.perl5.32 +++ b/pkg/docker/Dockerfile.perl5.32 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.php8.0 b/pkg/docker/Dockerfile.php8.0 index b8c9c046..c31513d4 100644 --- a/pkg/docker/Dockerfile.php8.0 +++ b/pkg/docker/Dockerfile.php8.0 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.python3.9 b/pkg/docker/Dockerfile.python3.9 index a0080fbb..76f50733 100644 --- a/pkg/docker/Dockerfile.python3.9 +++ b/pkg/docker/Dockerfile.python3.9 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ diff --git a/pkg/docker/Dockerfile.ruby2.7 b/pkg/docker/Dockerfile.ruby2.7 index ad3eec68..aa823756 100644 --- a/pkg/docker/Dockerfile.ruby2.7 +++ b/pkg/docker/Dockerfile.ruby2.7 @@ -8,7 +8,7 @@ RUN set -ex \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && hg clone https://hg.nginx.org/unit \ && cd unit \ - && hg up 1.21.0 \ + && hg up 1.22.0 \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ -- cgit From 774a6034d9daf32ac6c98da7e4c0ca9e820536b4 Mon Sep 17 00:00:00 2001 From: Valentin Bartenev Date: Thu, 4 Feb 2021 18:29:31 +0300 Subject: Added tag 1.22.0 for changeset 331bdadeca30 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 89fff74d..ca8e46f9 100644 --- a/.hgtags +++ b/.hgtags @@ -28,3 +28,4 @@ b391df5f0102aa6afe660cfc863729c1b1111c9e 1.12.0 86cdf66f82745d8db35345368dcdb38c79a4f03a 1.19.0 0e985b30067380782125f1c479eda4ef909418df 1.20.0 f804aaf7eee10a7d8116820840d6312dd4914a41 1.21.0 +331bdadeca30a49dd11b86af99124c2ffeb22d05 1.22.0 -- cgit