diff options
148 files changed, 6838 insertions, 3031 deletions
@@ -47,3 +47,4 @@ de07e42484ecc595050fcbd3581a64cc6b1c1de5 1.18.0-1 79f364e9aa907b1d7768e0e6686ce0a80fe61f44 1.19.0-1 0e985b30067380782125f1c479eda4ef909418df 1.20.0 29efab062b4e5d8a34b13724feb05c1890d738e8 1.20.0-1 +f804aaf7eee10a7d8116820840d6312dd4914a41 1.21.0 @@ -1,4 +1,50 @@ +Changes with Unit 1.21.0 19 Nov 2020 + + *) Change: procfs is mounted by default for all languages when "rootfs" + isolation is used. + + *) Change: any characters valid according to RFC 7230 are now allowed in + HTTP header field names. + + *) Change: HTTP header fields with underscores ("_") are now discarded + from requests by default. + + *) Feature: optional multithreaded request processing for Java, Python, + Perl, and Ruby apps. + + *) Feature: regular expressions in route matching patterns. + + *) Feature: compatibility with Python 3.9. + + *) Feature: the Python module now supports ASGI 2.0 legacy applications. + + *) Feature: the "protocol" option in Python applications aids choice + between ASGI and WSGI. + + *) Feature: the fastcgi_finish_request() PHP function that finalizes + request processing and continues code execution without holding onto + the client connection. + + *) Feature: the "discard_unsafe_fields" HTTP option that enables + discarding request header fields with irregular (but still valid) + characters in the field name. + + *) Feature: the "procfs" and "tmpfs" automount isolation options to + disable automatic mounting of eponymous filesystems. + + *) Bugfix: the router process could crash when running Go applications + under high load; the bug had appeared in 1.19.0. + + *) Bugfix: some language dependencies could remain mounted after using + "rootfs" isolation. + + *) Bugfix: various compatibility issues in Java applications. + + *) Bugfix: the Java module built with the musl C library couldn't run + applications that use "rootfs" isolation. + + Changes with Unit 1.20.0 08 Oct 2020 *) Change: the PHP module is now initialized before chrooting; this @@ -35,6 +35,8 @@ cat << END --no-ipv6 disable IPv6 support --no-unix-sockets disable Unix domain sockets support + --no-regex disable regular expression support + --no-pcre2 force using PCRE library --openssl enable OpenSSL library usage @@ -379,6 +379,7 @@ libunit-install: $NXT_BUILD_DIR/$NXT_LIB_UNIT_STATIC src/nxt_unit_sptr.h \ src/nxt_unit_typedefs.h \ src/nxt_unit_websocket.h \ + $NXT_BUILD_DIR/nxt_auto_config.h \ $NXT_BUILD_DIR/nxt_version.h \ src/nxt_websocket_header.h \ \$(DESTDIR)$NXT_INCDIR/ @@ -393,6 +394,7 @@ libunit-uninstall: \$(DESTDIR)$NXT_INCDIR/nxt_unit_sptr.h \ \$(DESTDIR)$NXT_INCDIR/nxt_unit_typedefs.h \ \$(DESTDIR)$NXT_INCDIR/nxt_unit_websocket.h \ + \$(DESTDIR)$NXT_INCDIR/nxt_auto_config.h \ \$(DESTDIR)$NXT_INCDIR/nxt_version.h \ \$(DESTDIR)$NXT_INCDIR/nxt_websocket_header.h @rmdir -p \$(DESTDIR)$NXT_INCDIR 2>/dev/null || true diff --git a/auto/modules/java b/auto/modules/java index be8f443c..60415c35 100644 --- a/auto/modules/java +++ b/auto/modules/java @@ -178,7 +178,7 @@ fi NXT_JAVA_LIB_SERVER_PATH="${NXT_JAVA_LIB_PATH}/server" -NXT_JAVA_LDFLAGS="-L${NXT_JAVA_LIB_SERVER_PATH} -Wl,-rpath ${NXT_JAVA_LIB_SERVER_PATH} -ljvm" +NXT_JAVA_LDFLAGS="-L${NXT_JAVA_LIB_SERVER_PATH} -Wl,-rpath,${NXT_JAVA_LIB_SERVER_PATH} -ljvm" nxt_found=no @@ -238,7 +238,7 @@ cat << END > $NXT_JAVA_JARS static const char *nxt_java_system_jars[] = { END -NXT_TOMCAT_VERSION=9.0.13 +NXT_TOMCAT_VERSION=9.0.39 NXT_JAR_VERSION=$NXT_TOMCAT_VERSION @@ -271,7 +271,7 @@ NXT_JAR_NAME=tomcat-util . auto/modules/java_get_jar NXT_JAR_NAME=ecj -NXT_JAR_VERSION=3.13.102 +NXT_JAR_VERSION=3.23.0 NXT_JAR_NAMESPACE=org/eclipse/jdt/ . auto/modules/java_get_jar @@ -284,7 +284,7 @@ static const char *nxt_java_unit_jars[] = { "$NXT_UNIT_JAR", END -NXT_JAR_VERSION=9.4.12.v20180830 +NXT_JAR_VERSION=9.4.33.v20201020 NXT_JAR_NAMESPACE=org/eclipse/jetty/ NXT_JAR_NAME=jetty-util @@ -297,7 +297,7 @@ NXT_JAR_NAME=jetty-http . auto/modules/java_get_jar NXT_JAR_NAME=classgraph -NXT_JAR_VERSION=4.4.11 +NXT_JAR_VERSION=4.8.90 NXT_JAR_NAMESPACE=io/github/classgraph/ . auto/modules/java_get_jar @@ -313,7 +313,7 @@ NXT_JAVA_LIBJVM="$NXT_JAVA_LIB_SERVER_PATH/libjvm.so" if [ "$NXT_SYSTEM" = "Darwin" ]; then NXT_JAVA_LIBC_DIR="/usr/lib" else -NXT_JAVA_LIBC_DIR=`ldd "$NXT_JAVA_LIBJVM" | grep libc.so | cut -d' ' -f3` +NXT_JAVA_LIBC_DIR=`ldd "$NXT_JAVA_LIBJVM" | grep -F libc. | cut -d' ' -f3` NXT_JAVA_LIBC_DIR=`dirname $NXT_JAVA_LIBC_DIR` fi @@ -326,11 +326,10 @@ cat << END > $NXT_BUILD_DIR/$NXT_JAVA_MOUNTS_HEADER static const nxt_fs_mount_t nxt_java_mounts[] = { - {(u_char *) "proc", (u_char *) "/proc", (u_char *) "proc", 0, NULL, 1}, {(u_char *) "$NXT_JAVA_LIBC_DIR", (u_char *) "$NXT_JAVA_LIBC_DIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_JAVA_HOME", (u_char *) "$NXT_JAVA_HOME", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, }; diff --git a/auto/modules/java_jar.sha512 b/auto/modules/java_jar.sha512 index 0f6daa8e..214cff33 100644 --- a/auto/modules/java_jar.sha512 +++ b/auto/modules/java_jar.sha512 @@ -1,14 +1,14 @@ -21cf5171c84fb12d0903d4f7d4f62e8b3dc60142ca1c717c46f7e09f6d40ea9a05a5bca34d468e00a00b427991de966fb060dcc282d532ee6a21567f802abfab classgraph-4.4.11.jar -7287b1ea3e18423d027a99ce40ae72e46e1700a65b474d2ec09af6a17b10653b7c2e69e9bb87efe14f4c593dc66b6370ea566fce90edb4b4190a903046817e6f ecj-3.13.102.jar -4626b970aa4d04422db93a4847eb9749768042255e0ba4d4944e1a8ca854de9e5eb093fbf5f0c05b122b65885324a9afdb1819acfb936514848c0726537cd403 jetty-http-9.4.12.v20180830.jar -19c0ea335efd54f6758b64725b4938cd124e60856b8966e1c60d33a5ebe3c62eea5babe5974c3a1b2c5ea49013ba5fc99aaa1a27e5a9c85e46f693fc679e5309 jetty-server-9.4.12.v20180830.jar -37ab6c29e925138d09a99bba9ead16b693318d8e098f1cf19fb56438c5d96479410628a84fa5a6c410850226acf1542aeab2a4894cebc9af8afa021c767d71f0 jetty-util-9.4.12.v20180830.jar -bf57568311fceb52f6611a2284d395cf181b634e4b44179766201f27b5a8c6981339aa35b3a0cb270d81d2a026e3ee724912040e0df5cf5ffd6442588b5b1e49 tomcat-api-9.0.13.jar -68238e5e00c7d0f0b159950a3c7ad6f666b343f7b31c5f0349a1e3184dea9c96ef50def82cc1025d6bcd4bc564d36306a169f96be7677185585ea2469b84a128 tomcat-el-api-9.0.13.jar -51c40eca728a34a96b2af0891355733e5ab5fc3ac5eec62f57e1ea905426b573bf9d55637d3b831838694054fc4d9dd06e48eb98eca57584d2750651bd286e0b tomcat-jasper-9.0.13.jar -ec64d3796a7f7224b451659f7f2b4a48da8a63da46557934d82c07420fa237a23c077f94908caa557ef51776d9e0a98b75cf43e6639c07f26aedb1ed4be99a62 tomcat-jasper-el-9.0.13.jar -2b5b92269c92e981268e346dc1484ebf4c7e481c2be26d90b02f650f94097bc8b9f9f1bc6579896e036a69747f41bf8493b3fa8d7626beaba95b28ae77f0f8ab tomcat-jsp-api-9.0.13.jar -f97891d80a6f96e9c10e5aa1d6a917961307f1a523266f4b0b857bb9a9bbe13ad09352a52e287d2dff84d18948ca25908ff13689eaea5bd806053c87822a892d tomcat-juli-9.0.13.jar -6579b30d95fa104663e2dced86115593868f18448f38235d88ccb003eeeeb61b49532300a30e7437e5c709891601f6b9909c33f1f7bcdf93ac118b67a742fdf1 tomcat-servlet-api-9.0.13.jar -af5d2f0209b7977e3d2ff6bb87dfa72aaa6684e9cd1aab7e48b4f0d389549c052c4d42cf3bd4c08837d79738609a9dfd93c9fdcf740d7a0ae851d9f8523ff65d tomcat-util-9.0.13.jar -18fb1a0f7e4ceb3416fc005e14f2eff6c49422806df03dfdd2a16ecc1292d55d59645707d408853573294446c2f525572dc3c94d45d32856c5d040945cbe416d tomcat-util-scan-9.0.13.jar +083f2102a50f2c3a4fc3993d328ddee17707b0529fdcee4868bd3b6901ff20a3d97e9733f83b2e46828469ecc273a1a3a5da26e4ba3d63f725a81b0b9e3e49e6 classgraph-4.8.90.jar +5e63e06eccd76a61249c7c6bd97e5b77dc37335bfdf316d941b00a16bae4f152d4306bac4e45c8422976187b0d77fe30cf7e6f6d4425f7732a3cb0cfa954385b ecj-3.23.0.jar +9abb4ba6802c39526b755429e3f41d641408f91f1c48844cd56cda116fb1479f05f1a3d90c18f30386b07d5593221c1f85ef565da0c6121b6ff005ec7d9c3bd9 jetty-http-9.4.33.v20201020.jar +e60e73ed3fd18e029c8e3cae04eac01d05f985dd0d9059eea620f4f9acac60799706d4b56b5a0f10cc33ff9272afdf11e712c604e31a77fa1afa1f95a57294ce jetty-server-9.4.33.v20201020.jar +35c5ad89d92e77f542d3ce42249f01c478706e2a013c5ab6a9504cf43520e3de90280d82a11389e9909060835421e3caada68e77d86df76a84856be60361d550 jetty-util-9.4.33.v20201020.jar +2d28d12196a7c2d342876d7e63c3a87fd613aadb32afad28d428d5dd9d9d0ebfa8300de3681d885455ac81d215326b11b845346416d5bd12d0e27c1763602caa tomcat-api-9.0.39.jar +661e613ae966e9b1fcc8815823c273fd7a05a6f8d3c469d86ce50650c1f350cbfb624fde8ee86235603078d9c5ccae5dfec8e03e10b5cb1e11109ae6ae4e85bb tomcat-el-api-9.0.39.jar +541d4794e1e15f26d7853cd7841f031e66fd3fc345102871cf58013591c8b57b6ab470d8ac28781c077f25e8d7fc06c19c6444db8e4a28ce4767ddbd907be004 tomcat-jasper-9.0.39.jar +57b916cb87f178604751f52f23f57fbd05da3001c9419602836e25d779727500bc1fceae195d5758aff28393b396751538f2ab7a9390738ebe4d0b4bd17b8ee4 tomcat-jasper-el-9.0.39.jar +d34deeecdecd59e40225ca3ad0ac197fc7f7845f09f551f5f17290ff9971741e62fac4bde16499d585b3d7c5574927c09740c0976dbddacecb19b4772f60c3e3 tomcat-jsp-api-9.0.39.jar +ee43f08f26e1d48c5cd3719e1eeec26e4ef389647a0bcd31435ed85c6c8770ab48fe287c60432aa8821b8fc8fce8ef0977a29b29c8c3b3cb246be53fb0182d1a tomcat-juli-9.0.39.jar +c387931fba85eae7ce030601dbd760cc7c2b0242a670b0c62321a8972e534076a40150044886053b3771099aa57c89057892d4f7fda84f65fa67b9c53dd51400 tomcat-servlet-api-9.0.39.jar +aa0b5ff10cdc686ba1dbc8a35d75aa42196126cbbde0d1ab1b9211ba57987a246bbe825c89a88104a0a60869fcbff4a8067bb247e42b69c887d6b48ee5c4a417 tomcat-util-9.0.39.jar +42c4b698cfb0a1fb154e290ac265c14693a9ef094e54116a324864f5b483f37d7a7decf67f7fe43b546cf1d2f25e9d763adc598a5e92979f756299e981dde43d tomcat-util-scan-9.0.39.jar diff --git a/auto/modules/php b/auto/modules/php index 41eeb1c3..e92a67cd 100644 --- a/auto/modules/php +++ b/auto/modules/php @@ -111,8 +111,7 @@ if /bin/sh -c "${NXT_PHP_CONFIG} --version" >> $NXT_AUTOCONF_ERR 2>&1; then if [ "$NXT_PHP_LIB_PATH" != "" ]; then # "php-config --ldflags" does not contain path to libphp, but # contains usually path to libraries required by extensions. - NXT_PHP_LDFLAGS="-L${NXT_PHP_LIB_PATH} \ - -Wl,-rpath ${NXT_PHP_LIB_PATH}" + NXT_PHP_LDFLAGS="-L${NXT_PHP_LIB_PATH} -Wl,-rpath,${NXT_PHP_LIB_PATH}" fi fi diff --git a/auto/modules/python b/auto/modules/python index 48f6e5ef..9be6b370 100644 --- a/auto/modules/python +++ b/auto/modules/python @@ -73,7 +73,7 @@ if /bin/sh -c "$NXT_PYTHON_CONFIG --prefix" >> $NXT_AUTOCONF_ERR 2>&1; then if [ "$NXT_PYTHON_LIB_PATH" != "" ]; then # "python-config --ldflags" may not contain path to libpython. - NXT_PYTHON_LDFLAGS="-L$NXT_PYTHON_LIB_PATH -Wl,-rpath $NXT_PYTHON_LIB_PATH" + NXT_PYTHON_LDFLAGS="-L$NXT_PYTHON_LIB_PATH -Wl,-rpath,$NXT_PYTHON_LIB_PATH" else NXT_PYTHON_LDFLAGS="" fi @@ -138,7 +138,7 @@ pyver = "python" + str(sys.version_info[0]) + "." + str(sys.version_info[1]) print("static const nxt_fs_mount_t nxt_python_mounts[] = {") -pattern = "{(u_char *) \"%s\", (u_char *) \"%s\", (u_char *) \"bind\", NXT_MS_BIND|NXT_MS_REC, NULL, 1}," +pattern = "{(u_char *) \"%s\", (u_char *) \"%s\", NXT_FS_BIND, (u_char *) \"bind\", 0, NULL, 1, 1}," base = None for p in sys.path: if len(p) > 0: diff --git a/auto/modules/ruby b/auto/modules/ruby index e0d54516..68324b44 100644 --- a/auto/modules/ruby +++ b/auto/modules/ruby @@ -156,23 +156,23 @@ cat << END > $NXT_RUBY_MOUNTS_PATH static const nxt_fs_mount_t nxt_ruby_mounts[] = { {(u_char *) "$NXT_RUBY_RUBYHDRDIR", (u_char *) "$NXT_RUBY_RUBYHDRDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_ARCHHDRDIR", (u_char *) "$NXT_RUBY_ARCHHDRDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_SITEDIR", (u_char *) "$NXT_RUBY_SITEDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_LIBDIR", (u_char *) "$NXT_RUBY_LIBDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_TOPDIR", (u_char *) "$NXT_RUBY_TOPDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, {(u_char *) "$NXT_RUBY_PREFIXDIR", (u_char *) "$NXT_RUBY_PREFIXDIR", - (u_char *) "bind", NXT_MS_BIND | NXT_MS_REC, NULL, 1}, + NXT_FS_BIND, (u_char *) "bind", 0, NULL, 1, 1}, END for path in `echo $NXT_RUBY_GEMPATH | tr ':' '\n'`; do $echo "{(u_char *) \"$path\", (u_char *) \"$path\"," >> $NXT_RUBY_MOUNTS_PATH - $echo "(u_char *) \"bind\", NXT_MS_BIND | NXT_MS_REC, NULL, 1}," >> $NXT_RUBY_MOUNTS_PATH + $echo "NXT_FS_BIND, (u_char *) \"bind\", 0, NULL, 1, 1}," >> $NXT_RUBY_MOUNTS_PATH done $echo "};" >> $NXT_RUBY_MOUNTS_PATH diff --git a/auto/options b/auto/options index d315b227..b6007bc2 100644 --- a/auto/options +++ b/auto/options @@ -16,8 +16,11 @@ NXT_DEBUG=NO NXT_INET6=YES NXT_UNIX_DOMAIN=YES -NXT_REGEX=NO -NXT_PCRE=NO +NXT_PCRE_CFLAGS= +NXT_PCRE_LIB= + +NXT_REGEX=YES +NXT_TRY_PCRE2=YES NXT_TLS=NO NXT_OPENSSL=NO @@ -73,7 +76,8 @@ do --no-ipv6) NXT_INET6=NO ;; --no-unix-sockets) NXT_UNIX_DOMAIN=NO ;; - --pcre) NXT_PCRE=YES ;; + --no-regex) NXT_REGEX=NO ;; + --no-pcre2) NXT_TRY_PCRE2=NO ;; --openssl) NXT_OPENSSL=YES ;; --gnutls) NXT_GNUTLS=YES ;; @@ -3,15 +3,41 @@ # Copyright (C) NGINX, Inc. -NXT_REGEX=NO -NXT_PCRE_CFLAGS= -NXT_PCRE_LIB= +nxt_found=no +NXT_HAVE_PCRE2=NO +if [ $NXT_TRY_PCRE2 = YES ]; then + if /bin/sh -c "(pcre2-config --version)" >> $NXT_AUTOCONF_ERR 2>&1; then -if [ $NXT_PCRE = YES ]; then + NXT_PCRE_CFLAGS=`pcre2-config --cflags` + NXT_PCRE_LIB=`pcre2-config --libs8` - nxt_found=no + nxt_feature="PCRE2 library" + nxt_feature_name=NXT_HAVE_PCRE2 + nxt_feature_run=no + nxt_feature_incs="-DPCRE2_CODE_UNIT_WIDTH=8 $NXT_PCRE_CFLAGS" + nxt_feature_libs=$NXT_PCRE_LIB + nxt_feature_test="#include <pcre2.h> + + int main(void) { + pcre2_code *re; + + re = pcre2_compile((PCRE2_SPTR)\"\", + PCRE2_ZERO_TERMINATED, 0, + NULL, NULL, NULL); + return (re == NULL); + }" + + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_PCRE2=YES + $echo " + PCRE2 version: `pcre2-config --version`" + fi + fi +fi +if [ $nxt_found = no ]; then if /bin/sh -c "(pcre-config --version)" >> $NXT_AUTOCONF_ERR 2>&1; then NXT_PCRE_CFLAGS=`pcre-config --cflags` @@ -33,17 +59,19 @@ if [ $NXT_PCRE = YES ]; then return 0; }" . auto/feature - fi - if [ $nxt_found = no ]; then - $echo - $echo $0: error: no PCRE library found. - $echo - exit 1; + if [ $nxt_found = yes ]; then + $echo " + PCRE version: `pcre-config --version`" + fi fi +fi - NXT_REGEX=YES - nxt_have=NXT_REGEX . auto/have +if [ $nxt_found = yes ]; then + nxt_have=NXT_HAVE_REGEX . auto/have - $echo " + PCRE version: `pcre-config --version`" +else + $echo + $echo $0: error: no PCRE library found. + $echo + exit 1; fi diff --git a/auto/sources b/auto/sources index e44dc4bb..01fec6c1 100644 --- a/auto/sources +++ b/auto/sources @@ -128,6 +128,9 @@ NXT_LIB_GNUTLS_SRCS="src/nxt_gnutls.c" NXT_LIB_CYASSL_SRCS="src/nxt_cyassl.c" NXT_LIB_POLARSSL_SRCS="src/nxt_polarssl.c" +NXT_LIB_PCRE_SRCS="src/nxt_pcre.c" +NXT_LIB_PCRE2_SRCS="src/nxt_pcre2.c" + NXT_LIB_EPOLL_SRCS="src/nxt_epoll_engine.c" NXT_LIB_KQUEUE_SRCS="src/nxt_kqueue_engine.c" NXT_LIB_EVENTPORT_SRCS="src/nxt_eventport_engine.c" @@ -211,6 +214,14 @@ if [ $NXT_POLARSSL = YES ]; then fi +if [ "$NXT_REGEX" = "YES" ]; then + if [ "$NXT_HAVE_PCRE2" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE2_SRCS" + else + NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE_SRCS" + fi +fi + if [ "$NXT_HAVE_EPOLL" = "YES" -o "$NXT_TEST_BUILD_EPOLL" = "YES" ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_EPOLL_SRCS" fi @@ -127,7 +127,11 @@ NXT_LIBRT= . auto/unix . auto/os/conf . auto/ssltls -. auto/pcre + +if [ $NXT_REGEX = YES ]; then + . auto/pcre +fi + . auto/isolation . auto/capability diff --git a/docs/changes.xml b/docs/changes.xml index 2a954dbe..83bb4a56 100644 --- a/docs/changes.xml +++ b/docs/changes.xml @@ -5,6 +5,170 @@ <change_log title="unit"> +<changes apply="unit-jsc15" ver="1.21.0" rev="1" + date="2020-11-19" time="18:00:00 +0300" + packager="Andrei Belov <defan@nginx.com>"> + +<change> +<para> +Initial release of Java 15 module for NGINX Unit. +</para> +</change> + +</changes> + + +<changes apply="unit-jsc14" ver="1.21.0" rev="1" + date="2020-11-19" time="18:00:00 +0300" + packager="Andrei Belov <defan@nginx.com>"> + +<change> +<para> +Initial release of Java 14 module for NGINX Unit. +</para> +</change> + +</changes> + + +<changes apply="unit-jsc13" ver="1.21.0" rev="1" + date="2020-11-19" time="18:00:00 +0300" + packager="Andrei Belov <defan@nginx.com>"> + +<change> +<para> +Initial release of Java 13 module for NGINX Unit. +</para> +</change> + +</changes> + + +<changes apply="unit-php + unit-python unit-python2.7 + unit-python3.4 unit-python3.5 unit-python3.6 unit-python3.7 + unit-python3.8 + unit-go + unit-perl + unit-ruby + unit-jsc-common unit-jsc8 unit-jsc10 unit-jsc11" + ver="1.21.0" rev="1" + date="2020-11-19" time="18:00:00 +0300" + packager="Andrei Belov <defan@nginx.com>"> + +<change> +<para> +NGINX Unit updated to 1.21.0. +</para> +</change> + +</changes> + + +<changes apply="unit" ver="1.21.0" rev="1" + date="2020-11-19" time="18:00:00 +0300" + packager="Andrei Belov <defan@nginx.com>"> + +<change type="change"> +<para> +procfs is mounted by default for all languages when "rootfs" isolation is used. +</para> +</change> + +<change type="change"> +<para> +any characters valid according to RFC 7230 are now allowed in HTTP header field +names. +</para> +</change> + +<change type="change"> +<para> +HTTP header fields with underscores ("_") are now discarded from requests by +default. +</para> +</change> + +<change type="feature"> +<para> +optional multithreaded request processing for Java, Python, Perl, and Ruby apps. +</para> +</change> + +<change type="feature"> +<para> +regular expressions in route matching patterns. +</para> +</change> + +<change type="feature"> +<para> +compatibility with Python 3.9. +</para> +</change> + +<change type="feature"> +<para> +the Python module now supports ASGI 2.0 legacy applications. +</para> +</change> + +<change type="feature"> +<para> +the "protocol" option in Python applications aids choice between ASGI and WSGI. +</para> +</change> + +<change type="feature"> +<para> +the fastcgi_finish_request() PHP function that finalizes request processing and +continues code execution without holding onto the client connection. +</para> +</change> + +<change type="feature"> +<para> +the "discard_unsafe_fields" HTTP option that enables discarding request header +fields with irregular (but still valid) characters in the field name. +</para> +</change> + +<change type="feature"> +<para> +the "procfs" and "tmpfs" automount isolation options to disable automatic +mounting of eponymous filesystems. +</para> +</change> + +<change type="bugfix"> +<para> +the router process could crash when running Go applications under high load; +the bug had appeared in 1.19.0. +</para> +</change> + +<change type="bugfix"> +<para> +some language dependencies could remain mounted after using "rootfs" isolation. +</para> +</change> + +<change type="bugfix"> +<para> +various compatibility issues in Java applications. +</para> +</change> + +<change type="bugfix"> +<para> +the Java module built with the musl C library couldn't run applications that +use "rootfs" isolation. +</para> +</change> + +</changes> + + <changes apply="unit-php unit-python unit-python2.7 unit-python3.4 unit-python3.5 unit-python3.6 unit-python3.7 diff --git a/go/nxt_cgo_lib.c b/go/nxt_cgo_lib.c index f7171f55..330697c1 100644 --- a/go/nxt_cgo_lib.c +++ b/go/nxt_cgo_lib.c @@ -10,16 +10,10 @@ #include <nxt_unit_request.h> -static void nxt_cgo_request_handler(nxt_unit_request_info_t *req); -static nxt_cgo_str_t *nxt_cgo_str_init(nxt_cgo_str_t *dst, - nxt_unit_sptr_t *sptr, uint32_t length); -static int nxt_cgo_add_port(nxt_unit_ctx_t *, nxt_unit_port_t *port); -static void nxt_cgo_remove_port(nxt_unit_t *, nxt_unit_port_t *port); static ssize_t nxt_cgo_port_send(nxt_unit_ctx_t *, nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size); static ssize_t nxt_cgo_port_recv(nxt_unit_ctx_t *, nxt_unit_port_t *port, void *buf, size_t buf_size, void *oob, size_t oob_size); -static void nxt_cgo_shm_ack_handler(nxt_unit_ctx_t *ctx); int nxt_cgo_run(uintptr_t handler) @@ -30,12 +24,12 @@ nxt_cgo_run(uintptr_t handler) memset(&init, 0, sizeof(init)); - init.callbacks.request_handler = nxt_cgo_request_handler; - init.callbacks.add_port = nxt_cgo_add_port; - init.callbacks.remove_port = nxt_cgo_remove_port; + init.callbacks.request_handler = nxt_go_request_handler; + init.callbacks.add_port = nxt_go_add_port; + init.callbacks.remove_port = nxt_go_remove_port; init.callbacks.port_send = nxt_cgo_port_send; init.callbacks.port_recv = nxt_cgo_port_recv; - init.callbacks.shm_ack_handler = nxt_cgo_shm_ack_handler; + init.callbacks.shm_ack_handler = nxt_go_shm_ack_handler; init.data = (void *) handler; @@ -52,76 +46,6 @@ nxt_cgo_run(uintptr_t handler) } -static void -nxt_cgo_request_handler(nxt_unit_request_info_t *req) -{ - uint32_t i; - uintptr_t go_req; - nxt_cgo_str_t method, uri, name, value, proto, host, remote_addr; - nxt_unit_field_t *f; - nxt_unit_request_t *r; - - r = req->request; - - go_req = nxt_go_request_create((uintptr_t) req, - nxt_cgo_str_init(&method, &r->method, r->method_length), - nxt_cgo_str_init(&uri, &r->target, r->target_length)); - - nxt_go_request_set_proto(go_req, - nxt_cgo_str_init(&proto, &r->version, r->version_length), 1, 1); - - for (i = 0; i < r->fields_count; i++) { - f = &r->fields[i]; - - nxt_go_request_add_header(go_req, - nxt_cgo_str_init(&name, &f->name, f->name_length), - nxt_cgo_str_init(&value, &f->value, f->value_length)); - } - - nxt_go_request_set_content_length(go_req, r->content_length); - nxt_go_request_set_host(go_req, - nxt_cgo_str_init(&host, &r->server_name, r->server_name_length)); - nxt_go_request_set_remote_addr(go_req, - nxt_cgo_str_init(&remote_addr, &r->remote, r->remote_length)); - - if (r->tls) { - nxt_go_request_set_tls(go_req); - } - - nxt_go_request_handler(go_req, (uintptr_t) req->unit->data); -} - - -static nxt_cgo_str_t * -nxt_cgo_str_init(nxt_cgo_str_t *dst, nxt_unit_sptr_t *sptr, uint32_t length) -{ - dst->length = length; - dst->start = nxt_unit_sptr_get(sptr); - - return dst; -} - - -static int -nxt_cgo_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) -{ - nxt_go_add_port((uintptr_t) ctx, port->id.pid, port->id.id, - port->in_fd, port->out_fd); - - port->in_fd = -1; - port->out_fd = -1; - - return NXT_UNIT_OK; -} - - -static void -nxt_cgo_remove_port(nxt_unit_t *unit, nxt_unit_port_t *port) -{ - nxt_go_remove_port(port->id.pid, port->id.id); -} - - static ssize_t nxt_cgo_port_send(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, const void *buf, size_t buf_size, const void *oob, size_t oob_size) @@ -140,78 +64,31 @@ nxt_cgo_port_recv(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, } -static void -nxt_cgo_shm_ack_handler(nxt_unit_ctx_t *ctx) -{ - return nxt_go_shm_ack_handler(); -} - - -int -nxt_cgo_response_create(uintptr_t req, int status, int fields, - uint32_t fields_size) -{ - return nxt_unit_response_init((nxt_unit_request_info_t *) req, - status, fields, fields_size); -} - - -int -nxt_cgo_response_add_field(uintptr_t req, uintptr_t name, uint8_t name_len, - uintptr_t value, uint32_t value_len) -{ - return nxt_unit_response_add_field((nxt_unit_request_info_t *) req, - (char *) name, name_len, - (char *) value, value_len); -} - - -int -nxt_cgo_response_send(uintptr_t req) -{ - return nxt_unit_response_send((nxt_unit_request_info_t *) req); -} - - ssize_t -nxt_cgo_response_write(uintptr_t req, uintptr_t start, uint32_t len) +nxt_cgo_response_write(nxt_unit_request_info_t *req, uintptr_t start, + uint32_t len) { - return nxt_unit_response_write_nb((nxt_unit_request_info_t *) req, - (void *) start, len, 0); + return nxt_unit_response_write_nb(req, (void *) start, len, 0); } ssize_t -nxt_cgo_request_read(uintptr_t req, uintptr_t dst, uint32_t dst_len) -{ - return nxt_unit_request_read((nxt_unit_request_info_t *) req, - (void *) dst, dst_len); -} - - -int -nxt_cgo_request_close(uintptr_t req) +nxt_cgo_request_read(nxt_unit_request_info_t *req, uintptr_t dst, + uint32_t dst_len) { - return 0; + return nxt_unit_request_read(req, (void *) dst, dst_len); } void -nxt_cgo_request_done(uintptr_t req, int res) +nxt_cgo_warn(const char *msg, uint32_t msg_len) { - nxt_unit_request_done((nxt_unit_request_info_t *) req, res); -} - - -void -nxt_cgo_unit_run_shared(uintptr_t ctx) -{ - nxt_unit_run_shared((nxt_unit_ctx_t *) ctx); + nxt_unit_warn(NULL, "%.*s", (int) msg_len, (char *) msg); } void -nxt_cgo_warn(uintptr_t msg, uint32_t msg_len) +nxt_cgo_alert(const char *msg, uint32_t msg_len) { - nxt_unit_warn(NULL, "%.*s", (int) msg_len, (char *) msg); + nxt_unit_alert(NULL, "%.*s", (int) msg_len, (char *) msg); } diff --git a/go/nxt_cgo_lib.h b/go/nxt_cgo_lib.h index fa515be5..3705d1ef 100644 --- a/go/nxt_cgo_lib.h +++ b/go/nxt_cgo_lib.h @@ -11,32 +11,22 @@ #include <stdint.h> #include <stdlib.h> #include <sys/types.h> +#include <nxt_unit.h> +#include <nxt_unit_request.h> -typedef struct { - int length; - char *start; -} nxt_cgo_str_t; +enum { + NXT_FIELDS_OFFSET = offsetof(nxt_unit_request_t, fields) +}; int nxt_cgo_run(uintptr_t handler); -int nxt_cgo_response_create(uintptr_t req, int code, int fields, - uint32_t fields_size); +ssize_t nxt_cgo_response_write(nxt_unit_request_info_t *req, + uintptr_t src, uint32_t len); -int nxt_cgo_response_add_field(uintptr_t req, uintptr_t name, uint8_t name_len, - uintptr_t value, uint32_t value_len); +ssize_t nxt_cgo_request_read(nxt_unit_request_info_t *req, + uintptr_t dst, uint32_t dst_len); -int nxt_cgo_response_send(uintptr_t req); - -ssize_t nxt_cgo_response_write(uintptr_t req, uintptr_t src, uint32_t len); - -ssize_t nxt_cgo_request_read(uintptr_t req, uintptr_t dst, uint32_t dst_len); - -int nxt_cgo_request_close(uintptr_t req); - -void nxt_cgo_request_done(uintptr_t req, int res); - -void nxt_cgo_unit_run_shared(uintptr_t ctx); - -void nxt_cgo_warn(uintptr_t msg, uint32_t msg_len); +void nxt_cgo_warn(const char *msg, uint32_t msg_len); +void nxt_cgo_alert(const char *msg, uint32_t msg_len); #endif /* _NXT_CGO_LIB_H_INCLUDED_ */ @@ -11,6 +11,7 @@ package unit import "C" import ( + "io" "net" "os" "sync" @@ -79,13 +80,13 @@ func getUnixConn(fd int) *net.UnixConn { c, err := net.FileConn(f) if err != nil { - nxt_go_warn("FileConn error %s", err) + nxt_go_alert("FileConn error %s", err) return nil } uc, ok := c.(*net.UnixConn) if !ok { - nxt_go_warn("Not a Unix-domain socket %d", fd) + nxt_go_alert("Not a Unix-domain socket %d", fd) return nil } @@ -93,30 +94,37 @@ func getUnixConn(fd int) *net.UnixConn { } //export nxt_go_add_port -func nxt_go_add_port(ctx C.uintptr_t, pid C.int, id C.int, rcv C.int, snd C.int) { - p := &port{ +func nxt_go_add_port(ctx *C.nxt_unit_ctx_t, p *C.nxt_unit_port_t) C.int { + + new_port := &port{ key: port_key{ - pid: int(pid), - id: int(id), + pid: int(p.id.pid), + id: int(p.id.id), }, - rcv: getUnixConn(int(rcv)), - snd: getUnixConn(int(snd)), + rcv: getUnixConn(int(p.in_fd)), + snd: getUnixConn(int(p.out_fd)), } - add_port(p) + add_port(new_port) + + p.in_fd = -1 + p.out_fd = -1 - if id == 65535 { - go func(ctx C.uintptr_t) { - C.nxt_cgo_unit_run_shared(ctx); + if new_port.key.id == 65535 { + go func(ctx *C.nxt_unit_ctx_t) { + C.nxt_unit_run_shared(ctx); }(ctx) } + + return C.NXT_UNIT_OK } //export nxt_go_remove_port -func nxt_go_remove_port(pid C.int, id C.int) { +func nxt_go_remove_port(unit *C.nxt_unit_t, p *C.nxt_unit_port_t) { + key := port_key{ - pid: int(pid), - id: int(id), + pid: int(p.id.pid), + id: int(p.id.id), } port_registry_.Lock() @@ -139,7 +147,7 @@ func nxt_go_port_send(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, p := find_port(key) if p == nil { - nxt_go_warn("port %d:%d not found", pid, id) + nxt_go_alert("port %d:%d not found", pid, id) return 0 } @@ -167,7 +175,7 @@ func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, p := find_port(key) if p == nil { - nxt_go_warn("port %d:%d not found", pid, id) + nxt_go_alert("port %d:%d not found", pid, id) return 0 } @@ -175,6 +183,12 @@ func nxt_go_port_recv(pid C.int, id C.int, buf unsafe.Pointer, buf_size C.int, GoBytes(oob, oob_size)) if err != nil { + if nerr, ok := err.(*net.OpError); ok { + if nerr.Err == io.EOF { + return 0 + } + } + nxt_go_warn("read result %d (%d), %s", n, oobn, err) n = -1 diff --git a/go/request.go b/go/request.go index 1d8c6702..7e2e848a 100644 --- a/go/request.go +++ b/go/request.go @@ -19,9 +19,9 @@ import ( ) type request struct { - req http.Request - resp *response - c_req C.uintptr_t + req http.Request + resp response + c_req *C.nxt_unit_request_info_t } func (r *request) Read(p []byte) (n int, err error) { @@ -35,110 +35,102 @@ func (r *request) Read(p []byte) (n int, err error) { } func (r *request) Close() error { - C.nxt_cgo_request_close(r.c_req) return nil } -func (r *request) response() *response { - if r.resp == nil { - r.resp = new_response(r.c_req, &r.req) - } +func new_request(c_req *C.nxt_unit_request_info_t) (r *request, err error) { + req := c_req.request - return r.resp -} + uri := GoStringN(&req.target, C.int(req.target_length)) -func (r *request) done() { - resp := r.response() - if !resp.headerSent { - resp.WriteHeader(http.StatusOK) + URL, err := url.ParseRequestURI(uri) + if err != nil { + return nil, err } - C.nxt_cgo_request_done(r.c_req, 0) -} - -func get_request(go_req uintptr) *request { - return (*request)(unsafe.Pointer(go_req)) -} - -//export nxt_go_request_create -func nxt_go_request_create(c_req C.uintptr_t, - c_method *C.nxt_cgo_str_t, c_uri *C.nxt_cgo_str_t) uintptr { - - uri := C.GoStringN(c_uri.start, c_uri.length) - var URL *url.URL - var err error - if URL, err = url.ParseRequestURI(uri); err != nil { - return 0 - } + proto := GoStringN(&req.version, C.int(req.version_length)) - r := &request{ - req: http.Request{ - Method: C.GoStringN(c_method.start, c_method.length), - URL: URL, - Header: http.Header{}, - Body: nil, + r = &request{ + req: http.Request { + URL: URL, + Header: http.Header{}, RequestURI: uri, + Method: GoStringN(&req.method, C.int(req.method_length)), + Proto: proto, + ProtoMajor: 1, + ProtoMinor: int(proto[7] - '0'), + ContentLength: int64(req.content_length), + Host: GoStringN(&req.server_name, C.int(req.server_name_length)), + RemoteAddr: GoStringN(&req.remote, C.int(req.remote_length)), }, + resp: response{header: http.Header{}, c_req: c_req}, c_req: c_req, } + r.req.Body = r - return uintptr(unsafe.Pointer(r)) -} + if req.tls != 0 { + r.req.TLS = &tls.ConnectionState{ } + r.req.URL.Scheme = "https" -//export nxt_go_request_set_proto -func nxt_go_request_set_proto(go_req uintptr, proto *C.nxt_cgo_str_t, - maj C.int, min C.int) { + } else { + r.req.URL.Scheme = "http" + } - r := get_request(go_req) - r.req.Proto = C.GoStringN(proto.start, proto.length) - r.req.ProtoMajor = int(maj) - r.req.ProtoMinor = int(min) -} + fields := get_fields(req) -//export nxt_go_request_add_header -func nxt_go_request_add_header(go_req uintptr, name *C.nxt_cgo_str_t, - value *C.nxt_cgo_str_t) { + for i := 0; i < len(fields); i++ { + f := &fields[i] - r := get_request(go_req) - r.req.Header.Add(C.GoStringN(name.start, name.length), - C.GoStringN(value.start, value.length)) -} + n := GoStringN(&f.name, C.int(f.name_length)) + v := GoStringN(&f.value, C.int(f.value_length)) -//export nxt_go_request_set_content_length -func nxt_go_request_set_content_length(go_req uintptr, l C.int64_t) { - get_request(go_req).req.ContentLength = int64(l) -} + r.req.Header.Add(n, v) + } -//export nxt_go_request_set_host -func nxt_go_request_set_host(go_req uintptr, host *C.nxt_cgo_str_t) { - get_request(go_req).req.Host = C.GoStringN(host.start, host.length) + return r, nil } -//export nxt_go_request_set_url -func nxt_go_request_set_url(go_req uintptr, scheme *C.char) { - get_request(go_req).req.URL.Scheme = C.GoString(scheme) -} +func get_fields(req *C.nxt_unit_request_t) []C.nxt_unit_field_t { + f := uintptr(unsafe.Pointer(req)) + uintptr(C.NXT_FIELDS_OFFSET) -//export nxt_go_request_set_remote_addr -func nxt_go_request_set_remote_addr(go_req uintptr, addr *C.nxt_cgo_str_t) { + h := &slice_header{ + Data: unsafe.Pointer(f), + Len: int(req.fields_count), + Cap: int(req.fields_count), + } - get_request(go_req).req.RemoteAddr = C.GoStringN(addr.start, addr.length) + return *(*[]C.nxt_unit_field_t)(unsafe.Pointer(h)) } -//export nxt_go_request_set_tls -func nxt_go_request_set_tls(go_req uintptr) { +//export nxt_go_request_handler +func nxt_go_request_handler(c_req *C.nxt_unit_request_info_t) { - get_request(go_req).req.TLS = &tls.ConnectionState{ } -} + go func(c_req *C.nxt_unit_request_info_t, handler http.Handler) { -//export nxt_go_request_handler -func nxt_go_request_handler(go_req uintptr, h uintptr) { - r := get_request(go_req) - handler := get_handler(h) - - go func(r *request) { - handler.ServeHTTP(r.response(), &r.req) - r.done() - }(r) + ctx := c_req.ctx + + for { + r, err := new_request(c_req) + + if err == nil { + handler.ServeHTTP(&r.resp, &r.req) + + if !r.resp.header_sent { + r.resp.WriteHeader(http.StatusOK) + } + + C.nxt_unit_request_done(c_req, C.NXT_UNIT_OK) + + } else { + C.nxt_unit_request_done(c_req, C.NXT_UNIT_ERROR) + } + + c_req = C.nxt_unit_dequeue_request(ctx) + if c_req == nil { + break + } + } + + }(c_req, get_handler(uintptr(c_req.unit.data))) } diff --git a/go/response.go b/go/response.go index bfa79656..a1af30b3 100644 --- a/go/response.go +++ b/go/response.go @@ -16,28 +16,17 @@ import ( type response struct { header http.Header - headerSent bool - req *http.Request - c_req C.uintptr_t + header_sent bool + c_req *C.nxt_unit_request_info_t ch chan int } -func new_response(c_req C.uintptr_t, req *http.Request) *response { - resp := &response{ - header: http.Header{}, - req: req, - c_req: c_req, - } - - return resp -} - func (r *response) Header() http.Header { return r.header } func (r *response) Write(p []byte) (n int, err error) { - if !r.headerSent { + if !r.header_sent { r.WriteHeader(http.StatusOK) } @@ -64,12 +53,11 @@ func (r *response) Write(p []byte) (n int, err error) { } func (r *response) WriteHeader(code int) { - if r.headerSent { - // Note: explicitly using Stderr, as Stdout is our HTTP output. + if r.header_sent { nxt_go_warn("multiple response.WriteHeader calls") return } - r.headerSent = true + r.header_sent = true // Set a default Content-Type if _, hasType := r.header["Content-Type"]; !hasType { @@ -86,21 +74,21 @@ func (r *response) WriteHeader(code int) { } } - C.nxt_cgo_response_create(r.c_req, C.int(code), C.int(fields), + C.nxt_unit_response_init(r.c_req, C.uint16_t(code), C.uint32_t(fields), C.uint32_t(fields_size)) for k, vv := range r.header { for _, v := range vv { - C.nxt_cgo_response_add_field(r.c_req, str_ref(k), C.uint8_t(len(k)), + C.nxt_unit_response_add_field(r.c_req, str_ref(k), C.uint8_t(len(k)), str_ref(v), C.uint32_t(len(v))) } } - C.nxt_cgo_response_send(r.c_req) + C.nxt_unit_response_send(r.c_req) } func (r *response) Flush() { - if !r.headerSent { + if !r.header_sent { r.WriteHeader(http.StatusOK) } } @@ -114,6 +102,6 @@ func wait_shm_ack(c chan int) { } //export nxt_go_shm_ack_handler -func nxt_go_shm_ack_handler() { +func nxt_go_shm_ack_handler(ctx *C.nxt_unit_ctx_t) { observer_registry_.notify(1) } @@ -30,15 +30,15 @@ func buf_ref(buf []byte) C.uintptr_t { return C.uintptr_t(uintptr(unsafe.Pointer(&buf[0]))) } -type StringHeader struct { +type string_header struct { Data unsafe.Pointer Len int } -func str_ref(s string) C.uintptr_t { - header := (*StringHeader)(unsafe.Pointer(&s)) +func str_ref(s string) *C.char { + header := (*string_header)(unsafe.Pointer(&s)) - return C.uintptr_t(uintptr(unsafe.Pointer(header.Data))) + return (*C.char)(header.Data) } func (buf *cbuf) init_bytes(b []byte) { @@ -46,12 +46,7 @@ func (buf *cbuf) init_bytes(b []byte) { buf.s = C.size_t(len(b)) } -func (buf *cbuf) init_string(s string) { - buf.b = str_ref(s) - buf.s = C.size_t(len(s)) -} - -type SliceHeader struct { +type slice_header struct { Data unsafe.Pointer Len int Cap int @@ -63,17 +58,17 @@ func (buf *cbuf) GoBytes() []byte { return b[:0] } - bytesHeader := &SliceHeader{ + header := &slice_header{ Data: unsafe.Pointer(uintptr(buf.b)), Len: int(buf.s), Cap: int(buf.s), } - return *(*[]byte)(unsafe.Pointer(bytesHeader)) + return *(*[]byte)(unsafe.Pointer(header)) } func GoBytes(buf unsafe.Pointer, size C.int) []byte { - bytesHeader := &SliceHeader{ + bytesHeader := &slice_header{ Data: buf, Len: int(size), Cap: int(size), @@ -82,12 +77,25 @@ func GoBytes(buf unsafe.Pointer, size C.int) []byte { return *(*[]byte)(unsafe.Pointer(bytesHeader)) } +func GoStringN(sptr *C.nxt_unit_sptr_t, l C.int) string { + p := unsafe.Pointer(sptr) + b := uintptr(p) + uintptr(*(*C.uint32_t)(p)) + + return C.GoStringN((*C.char)(unsafe.Pointer(b)), l) +} + func nxt_go_warn(format string, args ...interface{}) { str := fmt.Sprintf("[go] " + format, args...) C.nxt_cgo_warn(str_ref(str), C.uint32_t(len(str))) } +func nxt_go_alert(format string, args ...interface{}) { + str := fmt.Sprintf("[go] " + format, args...) + + C.nxt_cgo_alert(str_ref(str), C.uint32_t(len(str))) +} + type handler_registry struct { sync.RWMutex next uintptr diff --git a/pkg/deb/Makefile b/pkg/deb/Makefile index efeb642f..8a02105c 100644 --- a/pkg/deb/Makefile +++ b/pkg/deb/Makefile @@ -16,6 +16,21 @@ BUILD_DEPENDS = $(BUILD_DEPENDS_unit) MODULES= +# Ubuntu 20.10 +ifeq ($(CODENAME),groovy) +include Makefile.php +include Makefile.python27 +include Makefile.python38 +include Makefile.go +include Makefile.perl +include Makefile.ruby +include Makefile.jsc-common +include Makefile.jsc11 +include Makefile.jsc13 +include Makefile.jsc14 +include Makefile.jsc15 +endif + # Ubuntu 20.04 ifeq ($(CODENAME),focal) include Makefile.php diff --git a/pkg/deb/Makefile.jsc-common b/pkg/deb/Makefile.jsc-common index 928376c3..f7a6010b 100644 --- a/pkg/deb/Makefile.jsc-common +++ b/pkg/deb/Makefile.jsc-common @@ -6,7 +6,7 @@ MODULE_SUMMARY_jsc_common= Java shared packages for NGINX Unit MODULE_VERSION_jsc_common= $(VERSION) MODULE_RELEASE_jsc_common= 1 -ifneq (,$(findstring $(CODENAME),focal eoan disco buster)) +ifneq (,$(findstring $(CODENAME),groovy focal eoan disco buster)) JAVA_MINVERSION= 11 else JAVA_MINVERSION= 8 diff --git a/pkg/deb/Makefile.jsc13 b/pkg/deb/Makefile.jsc13 new file mode 100644 index 00000000..d22944dc --- /dev/null +++ b/pkg/deb/Makefile.jsc13 @@ -0,0 +1,71 @@ +MODULES+= jsc13 +MODULE_SUFFIX_jsc13= jsc13 + +MODULE_SUMMARY_jsc13= Java 13 module for NGINX Unit + +MODULE_VERSION_jsc13= $(VERSION) +MODULE_RELEASE_jsc13= 1 + +MODULE_CONFARGS_jsc13= java --module=java13 --home=/usr/lib/jvm/java-13-openjdk-$$\(DEB_HOST_ARCH\) --jars=/usr/share/unit-jsc-common/ +MODULE_MAKEARGS_jsc13= java13 +MODULE_INSTARGS_jsc13= java13-install + +MODULE_SOURCES_jsc13= unit.example-jsc-app \ + unit.example-jsc13-config + +BUILD_DEPENDS_jsc13= openjdk-13-jdk-headless openjdk-13-jre-headless +BUILD_DEPENDS+= $(BUILD_DEPENDS_jsc13) + +MODULE_BUILD_DEPENDS_jsc13=,openjdk-13-jdk-headless +MODULE_DEPENDS_jsc13=,openjdk-13-jre-headless,unit-jsc-common (= $(MODULE_VERSION_jsc_common)-$(MODULE_RELEASE_jsc_common)~$(CODENAME)) + +define MODULE_PREINSTALL_jsc13 + mkdir -p debian/unit-jsc13/usr/share/doc/unit-jsc13/examples/jsc-app + install -m 644 -p debian/unit.example-jsc-app debian/unit-jsc13/usr/share/doc/unit-jsc13/examples/jsc-app/index.jsp + install -m 644 -p debian/unit.example-jsc13-config debian/unit-jsc13/usr/share/doc/unit-jsc13/examples/unit.config + install -m 644 -p src/java/README.JSR-340 debian/unit-jsc13/usr/share/doc/unit-jsc13/ +endef +export MODULE_PREINSTALL_jsc13 + +define MODULE_POSTINSTALL_jsc13 + cd $$\(BUILDDIR_unit\) \&\& \ + DESTDIR=$$\(INSTALLDIR\) make java-shared-uninstall +endef +export MODULE_POSTINSTALL_jsc13 + +define MODULE_POST_jsc13 +cat <<BANNER +---------------------------------------------------------------------- + +The $(MODULE_SUMMARY_jsc13) has been installed. + +To check out the sample app, run these commands: + + sudo service unit restart + cd /usr/share/doc/unit-$(MODULE_SUFFIX_jsc13)/examples + sudo curl -X PUT --data-binary @unit.config --unix-socket /var/run/control.unit.sock http://localhost/config + curl http://localhost:8800/ + +Online documentation is available at https://unit.nginx.org + +NOTICE: + +This version of Unit code is made available in support of the open source +development process. This is an intermediate build made available for +testing purposes only. This Unit code is untested and presumed incompatible +with the JSR 340 Java Servlet 3.1 specification. You should not deploy or +write to this code. You should instead deploy and write production +applications on pre-built binaries that have been tested and certified +to meet the JSR-340 compatibility requirements such as certified binaries +published for the JSR-340 reference implementation available at +https://javaee.github.io/glassfish/. + +Redistribution of any Intermediate Build must retain this notice. + +Oracle and Java are registered trademarks of Oracle and/or its affiliates. +Other names may be trademarks of their respective owners. + +---------------------------------------------------------------------- +BANNER +endef +export MODULE_POST_jsc13 diff --git a/pkg/deb/Makefile.jsc14 b/pkg/deb/Makefile.jsc14 new file mode 100644 index 00000000..9b53a385 --- /dev/null +++ b/pkg/deb/Makefile.jsc14 @@ -0,0 +1,71 @@ +MODULES+= jsc14 +MODULE_SUFFIX_jsc14= jsc14 + +MODULE_SUMMARY_jsc14= Java 14 module for NGINX Unit + +MODULE_VERSION_jsc14= $(VERSION) +MODULE_RELEASE_jsc14= 1 + +MODULE_CONFARGS_jsc14= java --module=java14 --home=/usr/lib/jvm/java-14-openjdk-$$\(DEB_HOST_ARCH\) --jars=/usr/share/unit-jsc-common/ +MODULE_MAKEARGS_jsc14= java14 +MODULE_INSTARGS_jsc14= java14-install + +MODULE_SOURCES_jsc14= unit.example-jsc-app \ + unit.example-jsc14-config + +BUILD_DEPENDS_jsc14= openjdk-14-jdk-headless openjdk-14-jre-headless +BUILD_DEPENDS+= $(BUILD_DEPENDS_jsc14) + +MODULE_BUILD_DEPENDS_jsc14=,openjdk-14-jdk-headless +MODULE_DEPENDS_jsc14=,openjdk-14-jre-headless,unit-jsc-common (= $(MODULE_VERSION_jsc_common)-$(MODULE_RELEASE_jsc_common)~$(CODENAME)) + +define MODULE_PREINSTALL_jsc14 + mkdir -p debian/unit-jsc14/usr/share/doc/unit-jsc14/examples/jsc-app + install -m 644 -p debian/unit.example-jsc-app debian/unit-jsc14/usr/share/doc/unit-jsc14/examples/jsc-app/index.jsp + install -m 644 -p debian/unit.example-jsc14-config debian/unit-jsc14/usr/share/doc/unit-jsc14/examples/unit.config + install -m 644 -p src/java/README.JSR-340 debian/unit-jsc14/usr/share/doc/unit-jsc14/ +endef +export MODULE_PREINSTALL_jsc14 + +define MODULE_POSTINSTALL_jsc14 + cd $$\(BUILDDIR_unit\) \&\& \ + DESTDIR=$$\(INSTALLDIR\) make java-shared-uninstall +endef +export MODULE_POSTINSTALL_jsc14 + +define MODULE_POST_jsc14 +cat <<BANNER +---------------------------------------------------------------------- + +The $(MODULE_SUMMARY_jsc14) has been installed. + +To check out the sample app, run these commands: + + sudo service unit restart + cd /usr/share/doc/unit-$(MODULE_SUFFIX_jsc14)/examples + sudo curl -X PUT --data-binary @unit.config --unix-socket /var/run/control.unit.sock http://localhost/config + curl http://localhost:8800/ + +Online documentation is available at https://unit.nginx.org + +NOTICE: + +This version of Unit code is made available in support of the open source +development process. This is an intermediate build made available for +testing purposes only. This Unit code is untested and presumed incompatible +with the JSR 340 Java Servlet 3.1 specification. You should not deploy or +write to this code. You should instead deploy and write production +applications on pre-built binaries that have been tested and certified +to meet the JSR-340 compatibility requirements such as certified binaries +published for the JSR-340 reference implementation available at +https://javaee.github.io/glassfish/. + +Redistribution of any Intermediate Build must retain this notice. + +Oracle and Java are registered trademarks of Oracle and/or its affiliates. +Other names may be trademarks of their respective owners. + +---------------------------------------------------------------------- +BANNER +endef +export MODULE_POST_jsc14 diff --git a/pkg/deb/Makefile.jsc15 b/pkg/deb/Makefile.jsc15 new file mode 100644 index 00000000..10b5505b --- /dev/null +++ b/pkg/deb/Makefile.jsc15 @@ -0,0 +1,71 @@ +MODULES+= jsc15 +MODULE_SUFFIX_jsc15= jsc15 + +MODULE_SUMMARY_jsc15= Java 15 module for NGINX Unit + +MODULE_VERSION_jsc15= $(VERSION) +MODULE_RELEASE_jsc15= 1 + +MODULE_CONFARGS_jsc15= java --module=java15 --home=/usr/lib/jvm/java-15-openjdk-$$\(DEB_HOST_ARCH\) --jars=/usr/share/unit-jsc-common/ +MODULE_MAKEARGS_jsc15= java15 +MODULE_INSTARGS_jsc15= java15-install + +MODULE_SOURCES_jsc15= unit.example-jsc-app \ + unit.example-jsc15-config + +BUILD_DEPENDS_jsc15= openjdk-15-jdk-headless openjdk-15-jre-headless +BUILD_DEPENDS+= $(BUILD_DEPENDS_jsc15) + +MODULE_BUILD_DEPENDS_jsc15=,openjdk-15-jdk-headless +MODULE_DEPENDS_jsc15=,openjdk-15-jre-headless,unit-jsc-common (= $(MODULE_VERSION_jsc_common)-$(MODULE_RELEASE_jsc_common)~$(CODENAME)) + +define MODULE_PREINSTALL_jsc15 + mkdir -p debian/unit-jsc15/usr/share/doc/unit-jsc15/examples/jsc-app + install -m 644 -p debian/unit.example-jsc-app debian/unit-jsc15/usr/share/doc/unit-jsc15/examples/jsc-app/index.jsp + install -m 644 -p debian/unit.example-jsc15-config debian/unit-jsc15/usr/share/doc/unit-jsc15/examples/unit.config + install -m 644 -p src/java/README.JSR-340 debian/unit-jsc15/usr/share/doc/unit-jsc15/ +endef +export MODULE_PREINSTALL_jsc15 + +define MODULE_POSTINSTALL_jsc15 + cd $$\(BUILDDIR_unit\) \&\& \ + DESTDIR=$$\(INSTALLDIR\) make java-shared-uninstall +endef +export MODULE_POSTINSTALL_jsc15 + +define MODULE_POST_jsc15 +cat <<BANNER +---------------------------------------------------------------------- + +The $(MODULE_SUMMARY_jsc15) has been installed. + +To check out the sample app, run these commands: + + sudo service unit restart + cd /usr/share/doc/unit-$(MODULE_SUFFIX_jsc15)/examples + sudo curl -X PUT --data-binary @unit.config --unix-socket /var/run/control.unit.sock http://localhost/config + curl http://localhost:8800/ + +Online documentation is available at https://unit.nginx.org + +NOTICE: + +This version of Unit code is made available in support of the open source +development process. This is an intermediate build made available for +testing purposes only. This Unit code is untested and presumed incompatible +with the JSR 340 Java Servlet 3.1 specification. You should not deploy or +write to this code. You should instead deploy and write production +applications on pre-built binaries that have been tested and certified +to meet the JSR-340 compatibility requirements such as certified binaries +published for the JSR-340 reference implementation available at +https://javaee.github.io/glassfish/. + +Redistribution of any Intermediate Build must retain this notice. + +Oracle and Java are registered trademarks of Oracle and/or its affiliates. +Other names may be trademarks of their respective owners. + +---------------------------------------------------------------------- +BANNER +endef +export MODULE_POST_jsc15 diff --git a/pkg/deb/debian.module/unit.example-jsc13-config b/pkg/deb/debian.module/unit.example-jsc13-config new file mode 100644 index 00000000..ac0b1db6 --- /dev/null +++ b/pkg/deb/debian.module/unit.example-jsc13-config @@ -0,0 +1,15 @@ +{ + "applications": { + "example_java13": { + "processes": 1, + "type": "java 13", + "webapp": "/usr/share/doc/unit-jsc13/examples/jsc-app" + } + }, + + "listeners": { + "*:8800": { + "pass": "applications/example_java13" + } + } +} diff --git a/pkg/deb/debian.module/unit.example-jsc14-config b/pkg/deb/debian.module/unit.example-jsc14-config new file mode 100644 index 00000000..76109835 --- /dev/null +++ b/pkg/deb/debian.module/unit.example-jsc14-config @@ -0,0 +1,15 @@ +{ + "applications": { + "example_java14": { + "processes": 1, + "type": "java 14", + "webapp": "/usr/share/doc/unit-jsc14/examples/jsc-app" + } + }, + + "listeners": { + "*:8800": { + "pass": "applications/example_java14" + } + } +} diff --git a/pkg/deb/debian.module/unit.example-jsc15-config b/pkg/deb/debian.module/unit.example-jsc15-config new file mode 100644 index 00000000..9f9aec15 --- /dev/null +++ b/pkg/deb/debian.module/unit.example-jsc15-config @@ -0,0 +1,15 @@ +{ + "applications": { + "example_java15": { + "processes": 1, + "type": "java 15", + "webapp": "/usr/share/doc/unit-jsc15/examples/jsc-app" + } + }, + + "listeners": { + "*:8800": { + "pass": "applications/example_java15" + } + } +} diff --git a/pkg/docker/Dockerfile.full b/pkg/docker/Dockerfile.full index 724c24b6..a5f3e13f 100644 --- a/pkg/docker/Dockerfile.full +++ b/pkg/docker/Dockerfile.full @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.go1.11-dev b/pkg/docker/Dockerfile.go1.11-dev index 17b1d2e9..cb1c1d85 100644 --- a/pkg/docker/Dockerfile.go1.11-dev +++ b/pkg/docker/Dockerfile.go1.11-dev @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.jsc11 b/pkg/docker/Dockerfile.jsc11 index 65b8ce0f..f3d1d22d 100644 --- a/pkg/docker/Dockerfile.jsc11 +++ b/pkg/docker/Dockerfile.jsc11 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.minimal b/pkg/docker/Dockerfile.minimal index 5a10620a..da9c8712 100644 --- a/pkg/docker/Dockerfile.minimal +++ b/pkg/docker/Dockerfile.minimal @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.perl5.28 b/pkg/docker/Dockerfile.perl5.28 index 868527e5..976fa7b0 100644 --- a/pkg/docker/Dockerfile.perl5.28 +++ b/pkg/docker/Dockerfile.perl5.28 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.php7.3 b/pkg/docker/Dockerfile.php7.3 index b5789dec..f1178551 100644 --- a/pkg/docker/Dockerfile.php7.3 +++ b/pkg/docker/Dockerfile.php7.3 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.python2.7 b/pkg/docker/Dockerfile.python2.7 index 9bb83f7b..95241af6 100644 --- a/pkg/docker/Dockerfile.python2.7 +++ b/pkg/docker/Dockerfile.python2.7 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.python3.7 b/pkg/docker/Dockerfile.python3.7 index 99324844..52a5a257 100644 --- a/pkg/docker/Dockerfile.python3.7 +++ b/pkg/docker/Dockerfile.python3.7 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/pkg/docker/Dockerfile.ruby2.5 b/pkg/docker/Dockerfile.ruby2.5 index fef96867..342ccc9a 100644 --- a/pkg/docker/Dockerfile.ruby2.5 +++ b/pkg/docker/Dockerfile.ruby2.5 @@ -2,7 +2,7 @@ FROM debian:buster-slim LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>" -ENV UNIT_VERSION 1.20.0-1~buster +ENV UNIT_VERSION 1.21.0-1~buster RUN set -x \ && apt-get update \ diff --git a/src/java/nginx/unit/Context.java b/src/java/nginx/unit/Context.java index 5f7ec22f..0197858b 100644 --- a/src/java/nginx/unit/Context.java +++ b/src/java/nginx/unit/Context.java @@ -443,7 +443,7 @@ public class Context implements ServletContext, InitParams .enableClassInfo() .enableAnnotationInfo() //.enableSystemPackages() - .whitelistModules("javax.*") + .acceptModules("javax.*") //.enableAllInfo() ; @@ -1214,6 +1214,16 @@ public class Context implements ServletContext, InitParams processXmlInitParam(reg, (Element) child_node); continue; } + + if (tag_name.equals("filter-name") + || tag_name.equals("#text") + || tag_name.equals("#comment")) + { + continue; + } + + log("processWebXml: tag '" + tag_name + "' for filter '" + + filter_name + "' is ignored"); } filters_.add(reg); @@ -1306,6 +1316,22 @@ public class Context implements ServletContext, InitParams reg.setLoadOnStartup(Integer.parseInt(child_node.getTextContent().trim())); continue; } + + if (tag_name.equals("jsp-file")) { + reg.setJspFile(child_node.getTextContent().trim()); + continue; + } + + if (tag_name.equals("servlet-name") + || tag_name.equals("display-name") + || tag_name.equals("#text") + || tag_name.equals("#comment")) + { + continue; + } + + log("processWebXml: tag '" + tag_name + "' for servlet '" + + servlet_name + "' is ignored"); } servlets_.add(reg); @@ -1888,6 +1914,7 @@ public class Context implements ServletContext, InitParams private boolean initialized_ = false; private final List<FilterMap> filters_ = new ArrayList<>(); private boolean system_jsp_servlet_ = false; + private String jsp_file_; private MultipartConfigElement multipart_config_; public ServletReg(String name, Class<?> servlet_class) @@ -1921,6 +1948,21 @@ public class Context implements ServletContext, InitParams trace("ServletReg.init(): " + getName()); + if (jsp_file_ != null) { + setInitParameter("jspFile", jsp_file_); + jsp_file_ = null; + + ServletReg jsp_servlet = name2servlet_.get("jsp"); + + if (jsp_servlet.servlet_class_ != null) { + servlet_class_ = jsp_servlet.servlet_class_; + } else { + setClassName(jsp_servlet.getClassName()); + } + + system_jsp_servlet_ = jsp_servlet.system_jsp_servlet_; + } + if (system_jsp_servlet_) { JasperInitializer ji = new JasperInitializer(); @@ -1972,6 +2014,10 @@ public class Context implements ServletContext, InitParams throw new IllegalStateException("Class already initialized"); } + if (jsp_file_ != null) { + throw new IllegalStateException("jsp-file already initialized"); + } + super.setClassName(class_name); } @@ -1985,11 +2031,31 @@ public class Context implements ServletContext, InitParams throw new IllegalStateException("Class already initialized"); } + if (jsp_file_ != null) { + throw new IllegalStateException("jsp-file already initialized"); + } + super.setClassName(servlet_class.getName()); servlet_class_ = servlet_class; getAnnotationMultipartConfig(); } + public void setJspFile(String jsp_file) throws IllegalStateException + { + if (servlet_ != null + || servlet_class_ != null + || getClassName() != null) + { + throw new IllegalStateException("Class already initialized"); + } + + if (jsp_file_ != null) { + throw new IllegalStateException("jsp-file already initialized"); + } + + jsp_file_ = jsp_file; + } + private void getAnnotationMultipartConfig() { if (servlet_class_ == null) { return; diff --git a/src/java/nginx/unit/Response.java b/src/java/nginx/unit/Response.java index 099d7f15..268b359d 100644 --- a/src/java/nginx/unit/Response.java +++ b/src/java/nginx/unit/Response.java @@ -40,11 +40,13 @@ public class Response implements HttpServletResponse { private String characterEncoding = defaultCharacterEncoding; private String contentType = null; private String contentTypeHeader = null; + private Locale locale = null; private static final Charset ISO_8859_1 = StandardCharsets.ISO_8859_1; private static final Charset UTF_8 = StandardCharsets.UTF_8; private static final String CONTENT_TYPE = "Content-Type"; + private static final byte[] CONTENT_LANGUAGE_BYTES = "Content-Language".getBytes(ISO_8859_1); private static final byte[] SET_COOKIE_BYTES = "Set-Cookie".getBytes(ISO_8859_1); private static final byte[] EXPIRES_BYTES = "Expires".getBytes(ISO_8859_1); @@ -590,9 +592,13 @@ public class Response implements HttpServletResponse { @Override public Locale getLocale() { - log("getLocale"); + trace("getLocale"); - return null; + if (locale == null) { + return Locale.getDefault(); + } + + return locale; } @Override @@ -795,7 +801,16 @@ public class Response implements HttpServletResponse { @Override public void setLocale(Locale loc) { - log("setLocale: " + loc); + trace("setLocale: " + loc); + + if (loc == null || isCommitted()) { + return; + } + + locale = loc; + String lang = locale.toString().replace('_', '-'); + + setHeader(req_info_ptr, CONTENT_LANGUAGE_BYTES, lang.getBytes(ISO_8859_1)); } private void log(String msg) diff --git a/src/nodejs/unit-http/unit.cpp b/src/nodejs/unit-http/unit.cpp index 1ee5b742..c5bca49a 100644 --- a/src/nodejs/unit-http/unit.cpp +++ b/src/nodejs/unit-http/unit.cpp @@ -307,6 +307,8 @@ Unit::close_handler(nxt_unit_request_info_t *req) } catch (exception &e) { nxt_unit_req_warn(req, "close_handler: %s", e.str); + nxt_unit_request_done(req, NXT_UNIT_ERROR); + return; } diff --git a/src/nxt_application.c b/src/nxt_application.c index 6935346c..5d58e60c 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -208,14 +208,14 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) mounts = module[i].mounts; size += mounts->nelts * nxt_length("{\"src\": \"\", \"dst\": \"\", " - "\"fstype\": \"\", \"flags\": , " - "\"data\": \"\"},"); + "\"type\": , \"name\": \"\", " + "\"flags\": , \"data\": \"\"},"); mnt = mounts->elts; for (j = 0; j < mounts->nelts; j++) { size += nxt_strlen(mnt[j].src) + nxt_strlen(mnt[j].dst) - + nxt_strlen(mnt[j].fstype) + NXT_INT_T_LEN + + nxt_strlen(mnt[j].name) + (2 * NXT_INT_T_LEN) + (mnt[j].data == NULL ? 0 : nxt_strlen(mnt[j].data)); } } @@ -242,9 +242,10 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) for (j = 0; j < mounts->nelts; j++) { p = nxt_sprintf(p, end, "{\"src\": \"%s\", \"dst\": \"%s\", " - "\"fstype\": \"%s\", \"flags\": %d, " + "\"name\": \"%s\", \"type\": %d, \"flags\": %d, " "\"data\": \"%s\"},", - mnt[j].src, mnt[j].dst, mnt[j].fstype, mnt[j].flags, + mnt[j].src, mnt[j].dst, mnt[j].name, mnt[j].type, + mnt[j].flags, mnt[j].data == NULL ? (u_char *) "" : mnt[j].data); } @@ -386,11 +387,13 @@ nxt_discovery_module(nxt_task_t *task, nxt_mp_t *mp, nxt_array_t *modules, goto fail; } - to->fstype = nxt_cstr_dup(mp, to->fstype, from->fstype); - if (nxt_slow_path(to->fstype == NULL)) { + to->name = nxt_cstr_dup(mp, to->name, from->name); + if (nxt_slow_path(to->name == NULL)) { goto fail; } + to->type = from->type; + if (from->data != NULL) { to->data = nxt_cstr_dup(mp, to->data, from->data); if (nxt_slow_path(to->data == NULL)) { @@ -723,7 +726,7 @@ nxt_unit_default_init(nxt_task_t *task, nxt_unit_init_t *init) init->read_port.id.pid = my_port->pid; init->read_port.id.id = my_port->id; init->read_port.in_fd = my_port->pair[0]; - init->read_port.out_fd = -1; + init->read_port.out_fd = my_port->pair[1]; init->log_fd = 2; diff --git a/src/nxt_application.h b/src/nxt_application.h index cb49a033..5632f56f 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -51,6 +51,9 @@ typedef struct { nxt_str_t path; nxt_str_t module; char *callable; + nxt_str_t protocol; + uint32_t threads; + uint32_t thread_stack_size; } nxt_python_app_conf_t; @@ -62,11 +65,14 @@ typedef struct { typedef struct { char *script; + uint32_t threads; + uint32_t thread_stack_size; } nxt_perl_app_conf_t; typedef struct { nxt_str_t script; + uint32_t threads; } nxt_ruby_app_conf_t; @@ -75,6 +81,8 @@ typedef struct { char *webapp; nxt_conf_value_t *options; char *unit_jars; + uint32_t threads; + uint32_t thread_stack_size; } nxt_java_app_conf_t; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 4364057b..acb2e3de 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -11,6 +11,7 @@ #include <nxt_http.h> #include <nxt_sockaddr.h> #include <nxt_http_route_addr.h> +#include <nxt_regex.h> typedef enum { @@ -39,26 +40,34 @@ typedef enum { typedef nxt_int_t (*nxt_conf_vldt_handler_t)(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +typedef nxt_int_t (*nxt_conf_vldt_member_t)(nxt_conf_validation_t *vldt, + nxt_str_t *name, + nxt_conf_value_t *value); +typedef nxt_int_t (*nxt_conf_vldt_element_t)(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value); -typedef struct { - nxt_str_t name; - nxt_conf_vldt_type_t type:32; - nxt_conf_vldt_flags_t flags:32; - nxt_conf_vldt_handler_t validator; - void *data; -} nxt_conf_vldt_object_t; +typedef struct nxt_conf_vldt_object_s nxt_conf_vldt_object_t; +struct nxt_conf_vldt_object_s { + nxt_str_t name; + nxt_conf_vldt_type_t type:32; + nxt_conf_vldt_flags_t flags:32; + nxt_conf_vldt_handler_t validator; -#define NXT_CONF_VLDT_NEXT(f) { nxt_null_string, 0, 0, NULL, (f) } -#define NXT_CONF_VLDT_END { nxt_null_string, 0, 0, NULL, NULL } + union { + nxt_conf_vldt_object_t *members; + nxt_conf_vldt_object_t *next; + nxt_conf_vldt_member_t object; + nxt_conf_vldt_element_t array; + const char *string; + } u; +}; -typedef nxt_int_t (*nxt_conf_vldt_member_t)(nxt_conf_validation_t *vldt, - nxt_str_t *name, - nxt_conf_value_t *value); -typedef nxt_int_t (*nxt_conf_vldt_element_t)(nxt_conf_validation_t *vldt, - nxt_conf_value_t *value); +#define NXT_CONF_VLDT_NEXT(next) { .u.members = next } +#define NXT_CONF_VLDT_END { .name = nxt_null_string } + static nxt_int_t nxt_conf_vldt_type(nxt_conf_validation_t *vldt, nxt_str_t *name, nxt_conf_value_t *value, nxt_conf_vldt_type_t type); @@ -87,6 +96,12 @@ static nxt_int_t nxt_conf_vldt_return(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes_member(nxt_conf_validation_t *vldt, @@ -170,796 +185,718 @@ static nxt_int_t nxt_conf_vldt_clone_gidmap(nxt_conf_validation_t *vldt, nxt_conf_value_t *value); #endif -static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { - { nxt_string("read_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("keepalive_interval"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("max_frame_size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, + +static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[]; +#if (NXT_TLS) +static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[]; +#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[]; +#if (NXT_HAVE_ISOLATION_ROOTFS) +static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[]; +#endif + + +static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { + { + .name = nxt_string("settings"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_setting_members, + }, { + .name = nxt_string("listeners"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_listener, + }, { + .name = nxt_string("routes"), + .type = NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_routes, + }, { + .name = nxt_string("applications"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_app, + }, { + .name = nxt_string("upstreams"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_upstream, + }, { + .name = nxt_string("access_log"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { - { nxt_string("mime_types"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_mtypes, - NULL }, +static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { + { + .name = nxt_string("http"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_http_members, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { - { nxt_string("header_read_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("body_read_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("send_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("idle_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("body_buffer_size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("max_body_size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("body_temp_path"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("websocket"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_websocket_members }, - - { nxt_string("static"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_static_members }, + { + .name = nxt_string("header_read_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_read_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("send_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("idle_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_buffer_size"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("max_body_size"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("body_temp_path"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("discard_unsafe_fields"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("websocket"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_websocket_members, + }, { + .name = nxt_string("static"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_static_members, + }, NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[] = { - { nxt_string("http"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_http_members }, +static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[] = { + { + .name = nxt_string("read_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + + .name = nxt_string("keepalive_interval"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("max_frame_size"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_root_members[] = { - { nxt_string("settings"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_setting_members }, - - { nxt_string("listeners"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_listener }, - - { nxt_string("routes"), - NXT_CONF_VLDT_ARRAY | NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_routes, - NULL }, - - { nxt_string("applications"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_app }, - - { nxt_string("upstreams"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_upstream }, - - { nxt_string("access_log"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, +static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { + { + .name = nxt_string("mime_types"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_mtypes, + }, NXT_CONF_VLDT_END }; -#if (NXT_TLS) +static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { + { + .name = nxt_string("pass"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_pass, + }, { + .name = nxt_string("application"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_app_name, + }, -static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { - { nxt_string("certificate"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_certificate, - NULL }, +#if (NXT_TLS) + { + .name = nxt_string("tls"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_tls_members, + }, +#endif NXT_CONF_VLDT_END }; -#endif - - -static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { - { nxt_string("pass"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_pass, - NULL }, - - { nxt_string("application"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_app_name, - NULL }, #if (NXT_TLS) - { nxt_string("tls"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_tls_members }, +static nxt_conf_vldt_object_t nxt_conf_vldt_tls_members[] = { + { + .name = nxt_string("certificate"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_certificate, + }, + + NXT_CONF_VLDT_END +}; #endif + +static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { + { + .name = nxt_string("match"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_match_members, + }, { + .name = nxt_string("action"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_action, + }, + NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_match_members[] = { - { nxt_string("method"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns, - NULL }, - - { nxt_string("scheme"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_match_scheme_pattern, - NULL }, - - { nxt_string("host"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns, - NULL }, - - { nxt_string("source"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_addrs, - NULL }, - - { nxt_string("destination"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_addrs, - NULL }, - - { nxt_string("uri"), - NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_encoded_patterns, - NULL }, - - { nxt_string("arguments"), - NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_encoded_patterns_sets, - NULL }, - - { nxt_string("headers"), - NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns_sets, - NULL }, - - { nxt_string("cookies"), - NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_match_patterns_sets, - NULL }, + { + .name = nxt_string("method"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_patterns, + }, { + .name = nxt_string("scheme"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_match_scheme_pattern, + }, { + .name = nxt_string("host"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_patterns, + }, { + .name = nxt_string("source"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_addrs, + }, { + .name = nxt_string("destination"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_addrs, + }, { + .name = nxt_string("uri"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_encoded_patterns, + }, { + .name = nxt_string("arguments"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_encoded_patterns_sets, + }, { + .name = nxt_string("headers"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_patterns_sets, + }, { + .name = nxt_string("cookies"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_patterns_sets, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_pass_action_members[] = { - { nxt_string("pass"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_pass, - NULL }, + { + .name = nxt_string("pass"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_pass, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_return_action_members[] = { - { nxt_string("return"), - NXT_CONF_VLDT_INTEGER, - 0, - &nxt_conf_vldt_return, - NULL }, - - { nxt_string("location"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + { + .name = nxt_string("return"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_return, + }, { + .name = nxt_string("location"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_share_action_members[] = { - { nxt_string("share"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("fallback"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_action, - NULL }, + { + .name = nxt_string("share"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("fallback"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_action, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_proxy_action_members[] = { - { nxt_string("proxy"), - NXT_CONF_VLDT_STRING, - 0, - &nxt_conf_vldt_proxy, - NULL }, + { + .name = nxt_string("proxy"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_proxy, + }, NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_route_members[] = { - { nxt_string("match"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_match_members }, - - { nxt_string("action"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_action, - NULL }, +static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { + { + .name = nxt_string("executable"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("arguments"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_argument, + }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; -static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { - { nxt_string("timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("requests"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("shm"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, +static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { + { + .name = nxt_string("home"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("path"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("module"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("callable"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("protocol"), + .type = NXT_CONF_VLDT_STRING, + .validator = nxt_conf_vldt_python_protocol, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, { + .name = nxt_string("thread_stack_size"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_thread_stack_size, + }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) }; -static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { - { nxt_string("spare"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("max"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("idle_timeout"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, +static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { + { + .name = nxt_string("root"), + .type = NXT_CONF_VLDT_ANY_TYPE, + .validator = nxt_conf_vldt_php_targets_exclusive, + .u.string = "root", + }, { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_ANY_TYPE, + .validator = nxt_conf_vldt_php_targets_exclusive, + .u.string = "script", + }, { + .name = nxt_string("index"), + .type = NXT_CONF_VLDT_ANY_TYPE, + .validator = nxt_conf_vldt_php_targets_exclusive, + .u.string = "index", + }, { + .name = nxt_string("targets"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_php_targets, + }, - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_php_common_members) }; -static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { - -#if (NXT_HAVE_CLONE_NEWUSER) - { nxt_string("credential"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif - -#if (NXT_HAVE_CLONE_NEWPID) - { nxt_string("pid"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif - -#if (NXT_HAVE_CLONE_NEWNET) - { nxt_string("network"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { + { + .name = nxt_string("options"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_php_options_members, + }, -#if (NXT_HAVE_CLONE_NEWNS) - { nxt_string("mount"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) +}; -#if (NXT_HAVE_CLONE_NEWUTS) - { nxt_string("uname"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif -#if (NXT_HAVE_CLONE_NEWCGROUP) - { nxt_string("cgroup"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { + { + .name = nxt_string("file"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("admin"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_php_option, + }, { + .name = nxt_string("user"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_php_option, + }, NXT_CONF_VLDT_END }; -#if (NXT_HAVE_CLONE_NEWUSER) - -static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { - { nxt_string("container"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("host"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, - - { nxt_string("size"), - NXT_CONF_VLDT_INTEGER, - 0, - NULL, - NULL }, +static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { + { + .name = nxt_string("root"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("index"), + .type = NXT_CONF_VLDT_STRING, + }, NXT_CONF_VLDT_END }; -#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { + { + .name = nxt_string("root"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("index"), + .type = NXT_CONF_VLDT_STRING, + }, -#if (NXT_HAVE_ISOLATION_ROOTFS) - -static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { - { nxt_string("language_deps"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, - - NXT_CONF_VLDT_END + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_php_common_members) }; -#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { + { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, { + .name = nxt_string("thread_stack_size"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_thread_stack_size, + }, -static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { - { nxt_string("namespaces"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_clone_namespaces, - (void *) &nxt_conf_vldt_app_namespaces_members }, + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) +}; -#if (NXT_HAVE_CLONE_NEWUSER) - { nxt_string("uidmap"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_clone_uidmap }, +static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { + { + .name = nxt_string("script"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, - { nxt_string("gidmap"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_clone_gidmap }, + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) +}; -#endif -#if (NXT_HAVE_ISOLATION_ROOTFS) +static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { + { + .name = nxt_string("classpath"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_java_classpath, + }, { + .name = nxt_string("webapp"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + }, { + .name = nxt_string("options"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_java_option, + }, { + .name = nxt_string("unit_jars"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("threads"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_threads, + }, { + .name = nxt_string("thread_stack_size"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_thread_stack_size, + }, - { nxt_string("rootfs"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, + NXT_CONF_VLDT_NEXT(nxt_conf_vldt_common_members) +}; - { nxt_string("automount"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_app_automount_members }, -#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { + { + .name = nxt_string("type"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("limits"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_app_limits_members, + }, { + .name = nxt_string("processes"), + .type = NXT_CONF_VLDT_INTEGER | NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_processes, + .u.members = nxt_conf_vldt_app_processes_members, + }, { + .name = nxt_string("user"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("group"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("working_directory"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("environment"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_environment, + }, { + .name = nxt_string("isolation"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_isolation, + .u.members = nxt_conf_vldt_app_isolation_members, + }, -#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) + NXT_CONF_VLDT_END +}; - { nxt_string("new_privs"), - NXT_CONF_VLDT_BOOLEAN, - 0, - NULL, - NULL }, -#endif +static nxt_conf_vldt_object_t nxt_conf_vldt_app_limits_members[] = { + { + .name = nxt_string("timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("requests"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("shm"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_common_members[] = { - { nxt_string("type"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("limits"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_app_limits_members }, - - { nxt_string("processes"), - NXT_CONF_VLDT_INTEGER | NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_processes, - (void *) &nxt_conf_vldt_app_processes_members }, - - { nxt_string("user"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("group"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("working_directory"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("environment"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_environment }, - - { nxt_string("isolation"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_isolation, - (void *) &nxt_conf_vldt_app_isolation_members }, +static nxt_conf_vldt_object_t nxt_conf_vldt_app_processes_members[] = { + { + .name = nxt_string("spare"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("max"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("idle_timeout"), + .type = NXT_CONF_VLDT_INTEGER, + }, NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_external_members[] = { - { nxt_string("executable"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("arguments"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_argument }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; - +static nxt_conf_vldt_object_t nxt_conf_vldt_app_isolation_members[] = { + { + .name = nxt_string("namespaces"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_clone_namespaces, + .u.members = nxt_conf_vldt_app_namespaces_members, + }, -static nxt_conf_vldt_object_t nxt_conf_vldt_python_members[] = { - { nxt_string("home"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("path"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("module"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("callable"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; +#if (NXT_HAVE_CLONE_NEWUSER) + { + .name = nxt_string("uidmap"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_clone_uidmap, + }, { + .name = nxt_string("gidmap"), + .type = NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_array_iterator, + .u.array = nxt_conf_vldt_clone_gidmap, + }, +#endif +#if (NXT_HAVE_ISOLATION_ROOTFS) + { + .name = nxt_string("rootfs"), + .type = NXT_CONF_VLDT_STRING, + }, { + .name = nxt_string("automount"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_app_automount_members, + }, +#endif -static nxt_conf_vldt_object_t nxt_conf_vldt_php_target_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("index"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, +#if (NXT_HAVE_PR_SET_NO_NEW_PRIVS) + { + .name = nxt_string("new_privs"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_php_options_members[] = { - { nxt_string("file"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("admin"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_php_option }, - - { nxt_string("user"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_php_option }, - - NXT_CONF_VLDT_END -}; +static nxt_conf_vldt_object_t nxt_conf_vldt_app_namespaces_members[] = { +#if (NXT_HAVE_CLONE_NEWUSER) + { + .name = nxt_string("credential"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif -static nxt_conf_vldt_object_t nxt_conf_vldt_php_common_members[] = { - { nxt_string("options"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object, - (void *) &nxt_conf_vldt_php_options_members }, +#if (NXT_HAVE_CLONE_NEWPID) + { + .name = nxt_string("pid"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; +#if (NXT_HAVE_CLONE_NEWNET) + { + .name = nxt_string("network"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif +#if (NXT_HAVE_CLONE_NEWNS) + { + .name = nxt_string("mount"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif -static nxt_conf_vldt_object_t nxt_conf_vldt_php_notargets_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - { nxt_string("index"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) -}; +#if (NXT_HAVE_CLONE_NEWUTS) + { + .name = nxt_string("uname"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif +#if (NXT_HAVE_CLONE_NEWCGROUP) + { + .name = nxt_string("cgroup"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, +#endif -static nxt_conf_vldt_object_t nxt_conf_vldt_php_members[] = { - { nxt_string("root"), - NXT_CONF_VLDT_ANY_TYPE, - 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "root" }, - - { nxt_string("script"), - NXT_CONF_VLDT_ANY_TYPE, - 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "script" }, - - { nxt_string("index"), - NXT_CONF_VLDT_ANY_TYPE, - 0, - &nxt_conf_vldt_php_targets_exclusive, - (void *) "index" }, - - { nxt_string("targets"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_php_targets, - NULL }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_php_common_members) + NXT_CONF_VLDT_END }; -static nxt_conf_vldt_object_t nxt_conf_vldt_perl_members[] = { - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, +#if (NXT_HAVE_ISOLATION_ROOTFS) - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) +static nxt_conf_vldt_object_t nxt_conf_vldt_app_automount_members[] = { + { + .name = nxt_string("language_deps"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("tmpfs"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("procfs"), + .type = NXT_CONF_VLDT_BOOLEAN, + }, + + NXT_CONF_VLDT_END }; +#endif -static nxt_conf_vldt_object_t nxt_conf_vldt_ruby_members[] = { - { nxt_string("script"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) -}; +#if (NXT_HAVE_CLONE_NEWUSER) +static nxt_conf_vldt_object_t nxt_conf_vldt_app_procmap_members[] = { + { + .name = nxt_string("container"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("host"), + .type = NXT_CONF_VLDT_INTEGER, + }, { + .name = nxt_string("size"), + .type = NXT_CONF_VLDT_INTEGER, + }, -static nxt_conf_vldt_object_t nxt_conf_vldt_java_members[] = { - { nxt_string("classpath"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_java_classpath }, - - { nxt_string("webapp"), - NXT_CONF_VLDT_STRING, - NXT_CONF_VLDT_REQUIRED, - NULL, - NULL }, - - { nxt_string("options"), - NXT_CONF_VLDT_ARRAY, - 0, - &nxt_conf_vldt_array_iterator, - (void *) &nxt_conf_vldt_java_option }, - - { nxt_string("unit_jars"), - NXT_CONF_VLDT_STRING, - 0, - NULL, - NULL }, - - NXT_CONF_VLDT_NEXT(&nxt_conf_vldt_common_members) + NXT_CONF_VLDT_END }; +#endif + static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_members[] = { - { nxt_string("servers"), - NXT_CONF_VLDT_OBJECT, - 0, - &nxt_conf_vldt_object_iterator, - (void *) &nxt_conf_vldt_server }, + { + .name = nxt_string("servers"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object_iterator, + .u.object = nxt_conf_vldt_server, + }, NXT_CONF_VLDT_END }; static nxt_conf_vldt_object_t nxt_conf_vldt_upstream_server_members[] = { - { nxt_string("weight"), - NXT_CONF_VLDT_NUMBER, - 0, - &nxt_conf_vldt_server_weight, - NULL }, + { + .name = nxt_string("weight"), + .type = NXT_CONF_VLDT_NUMBER, + .validator = nxt_conf_vldt_server_weight, + }, NXT_CONF_VLDT_END }; @@ -1440,6 +1377,72 @@ nxt_conf_vldt_proxy(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, static nxt_int_t +nxt_conf_vldt_python_protocol(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + nxt_str_t proto; + + static const nxt_str_t wsgi = nxt_string("wsgi"); + static const nxt_str_t asgi = nxt_string("asgi"); + + nxt_conf_get_string(value, &proto); + + if (nxt_strstr_eq(&proto, &wsgi) || nxt_strstr_eq(&proto, &asgi)) { + return NXT_OK; + } + + return nxt_conf_vldt_error(vldt, "The \"protocol\" can either be " + "\"wsgi\" or \"asgi\"."); +} + + +static nxt_int_t +nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + int64_t threads; + + threads = nxt_conf_get_number(value); + + if (threads < 1) { + return nxt_conf_vldt_error(vldt, "The \"threads\" number must be " + "equal to or greater than 1."); + } + + if (threads > NXT_INT32_T_MAX) { + return nxt_conf_vldt_error(vldt, "The \"threads\" number must " + "not exceed %d.", NXT_INT32_T_MAX); + } + + return NXT_OK; +} + + +static nxt_int_t +nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + int64_t size; + + size = nxt_conf_get_number(value); + + if (size < PTHREAD_STACK_MIN) { + return nxt_conf_vldt_error(vldt, "The \"thread_stack_size\" number " + "must be equal to or greater than %d.", + PTHREAD_STACK_MIN); + } + + if ((size % nxt_pagesize) != 0) { + return nxt_conf_vldt_error(vldt, "The \"thread_stack_size\" number " + "must be a multiple of the system page size (%d).", + nxt_pagesize); + } + + return NXT_OK; +} + + +static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) { @@ -1502,8 +1505,12 @@ static nxt_int_t nxt_conf_vldt_match_pattern(nxt_conf_validation_t *vldt, nxt_conf_value_t *value) { - nxt_str_t pattern; - nxt_uint_t i, first, last; + nxt_str_t pattern; + nxt_uint_t i, first, last; +#if (NXT_HAVE_REGEX) + nxt_regex_t *re; + nxt_regex_err_t err; +#endif if (nxt_conf_type(value) != NXT_CONF_STRING) { return nxt_conf_vldt_error(vldt, "The \"match\" patterns for \"host\", " @@ -1517,6 +1524,32 @@ nxt_conf_vldt_match_pattern(nxt_conf_validation_t *vldt, } first = (pattern.start[0] == '!'); + + if (first < pattern.length && pattern.start[first] == '~') { +#if (NXT_HAVE_REGEX) + pattern.start += first + 1; + pattern.length -= first + 1; + + re = nxt_regex_compile(vldt->pool, &pattern, &err); + if (nxt_slow_path(re == NULL)) { + if (err.offset < pattern.length) { + return nxt_conf_vldt_error(vldt, "Invalid regular expression: " + "%s at offset %d", + err.msg, err.offset); + } + + return nxt_conf_vldt_error(vldt, "Invalid regular expression: %s", + err.msg); + } + + return NXT_OK; +#else + return nxt_conf_vldt_error(vldt, "Unit is built without support of " + "regular expressions: \"--no-regex\" " + "./configure option was set."); +#endif + } + last = pattern.length - 1; for (i = first; i < last; i++) { @@ -1887,8 +1920,8 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, for ( ;; ) { if (vals->name.length == 0) { - if (vals->data != NULL) { - vals = vals->data; + if (vals->u.members != NULL) { + vals = vals->u.members; continue; } @@ -1921,8 +1954,8 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, for ( ;; ) { if (vals->name.length == 0) { - if (vals->data != NULL) { - vals = vals->data; + if (vals->u.members != NULL) { + vals = vals->u.members; continue; } @@ -1942,7 +1975,7 @@ nxt_conf_vldt_object(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, } if (vals->validator != NULL) { - ret = vals->validator(vldt, member, vals->data); + ret = vals->validator(vldt, member, vals->u.members); if (ret != NXT_OK) { return ret; diff --git a/src/nxt_external.c b/src/nxt_external.c index 1adb839c..5703e294 100644 --- a/src/nxt_external.c +++ b/src/nxt_external.c @@ -101,18 +101,24 @@ nxt_external_start(nxt_task_t *task, nxt_process_data_t *data) return NXT_ERROR; } + rc = nxt_external_fd_no_cloexec(task, my_port->pair[1]); + if (nxt_slow_path(rc != NXT_OK)) { + return NXT_ERROR; + } + end = buf + sizeof(buf); p = nxt_sprintf(buf, end, "%s;%uD;" "%PI,%ud,%d;" "%PI,%ud,%d;" - "%PI,%ud,%d;" + "%PI,%ud,%d,%d;" "%d,%z,%Z", NXT_VERSION, my_port->process->stream, main_port->pid, main_port->id, main_port->pair[1], router_port->pid, router_port->id, router_port->pair[1], my_port->pid, my_port->id, my_port->pair[0], + my_port->pair[1], 2, conf->shm_limit); if (nxt_slow_path(p == end)) { diff --git a/src/nxt_fs.c b/src/nxt_fs.c index 0228c25a..71498f99 100644 --- a/src/nxt_fs.c +++ b/src/nxt_fs.c @@ -18,15 +18,59 @@ static nxt_int_t nxt_fs_mkdir(const u_char *dir, mode_t mode); nxt_int_t nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) { - int rc; + int rc; + const char *fsname; + unsigned long flags; - rc = mount((const char *) mnt->src, (const char *) mnt->dst, - (const char *) mnt->fstype, mnt->flags, mnt->data); + flags = 0; + + switch (mnt->type) { + case NXT_FS_BIND: + if (nxt_slow_path(mnt->flags != 0)) { + nxt_log(task, NXT_LOG_WARN, + "bind mount ignores additional flags"); + } + + fsname = "bind"; + flags = MS_BIND | MS_REC; + break; + + case NXT_FS_PROC: + fsname = "proc"; + goto getflags; + + case NXT_FS_TMP: + fsname = "tmpfs"; + goto getflags; + + default: + fsname = (const char *) mnt->name; + + getflags: + + if (mnt->flags & NXT_FS_FLAGS_NODEV) { + flags |= MS_NODEV; + } + + if (mnt->flags & NXT_FS_FLAGS_NOEXEC) { + flags |= MS_NOEXEC; + } + + if (mnt->flags & NXT_FS_FLAGS_NOSUID) { + flags |= MS_NOSUID; + } + + if (!(mnt->flags & NXT_FS_FLAGS_NOTIME)) { + flags |= MS_RELATIME; + } + } + + rc = mount((const char *) mnt->src, (const char *) mnt->dst, fsname, flags, + mnt->data); if (nxt_slow_path(rc < 0)) { - nxt_alert(task, "mount(\"%s\", \"%s\", \"%s\", %d, \"%s\") %E", - mnt->src, mnt->dst, mnt->fstype, mnt->flags, mnt->data, - nxt_errno); + nxt_alert(task, "mount(\"%s\", \"%s\", \"%s\", %ul, \"%s\") %E", + mnt->src, mnt->dst, fsname, flags, mnt->data, nxt_errno); return NXT_ERROR; } @@ -34,37 +78,66 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) return NXT_OK; } - #elif (NXT_HAVE_FREEBSD_NMOUNT) nxt_int_t nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) { + int flags; u_char *data, *p, *end; size_t iovlen; nxt_int_t ret; - const char *fstype; + const char *fsname; struct iovec iov[128]; char errmsg[256]; - if (nxt_strncmp(mnt->fstype, "bind", 4) == 0) { - fstype = "nullfs"; + if (nxt_slow_path((mnt->flags & NXT_FS_FLAGS_NODEV) && !mnt->builtin)) { + nxt_alert(task, "nmount(2) doesn't support \"nodev\" option"); - } else if (nxt_strncmp(mnt->fstype, "proc", 4) == 0) { - fstype = "procfs"; + return NXT_ERROR; + } - } else if (nxt_strncmp(mnt->fstype, "tmpfs", 5) == 0) { - fstype = "tmpfs"; + flags = 0; - } else { - nxt_alert(task, "mount type \"%s\" not implemented.", mnt->fstype); - return NXT_ERROR; + switch (mnt->type) { + case NXT_FS_BIND: + fsname = "nullfs"; + break; + + case NXT_FS_PROC: + fsname = "procfs"; + goto getflags; + + case NXT_FS_TMP: + fsname = "tmpfs"; + goto getflags; + + default: + fsname = (const char *) mnt->name; + + getflags: + + if (mnt->flags & NXT_FS_FLAGS_NOEXEC) { + flags |= MNT_NOEXEC; + } + + if (mnt->flags & NXT_FS_FLAGS_NOSUID) { + flags |= MNT_NOSUID; + } + + if (mnt->flags & NXT_FS_FLAGS_NOTIME) { + flags |= MNT_NOATIME; + } + + if (mnt->flags & NXT_FS_FLAGS_RDONLY) { + flags |= MNT_RDONLY; + } } iov[0].iov_base = (void *) "fstype"; iov[0].iov_len = 7; - iov[1].iov_base = (void *) fstype; - iov[1].iov_len = nxt_strlen(fstype) + 1; + iov[1].iov_base = (void *) fsname; + iov[1].iov_len = nxt_strlen(fsname) + 1; iov[2].iov_base = (void *) "fspath"; iov[2].iov_len = 7; iov[3].iov_base = (void *) mnt->dst; @@ -99,8 +172,10 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) *end = '\0'; - iov[iovlen++].iov_base = (void *) p; - iov[iovlen++].iov_len = (end - p) + 1; + iov[iovlen].iov_base = (void *) p; + iov[iovlen].iov_len = (end - p) + 1; + + iovlen++; p = end + 1; @@ -109,15 +184,17 @@ nxt_fs_mount(nxt_task_t *task, nxt_fs_mount_t *mnt) *end = '\0'; } - iov[iovlen++].iov_base = (void *) p; - iov[iovlen++].iov_len = nxt_strlen(p) + 1; + iov[iovlen].iov_base = (void *) p; + iov[iovlen].iov_len = nxt_strlen(p) + 1; + + iovlen++; } while (end != NULL && nxt_nitems(iov) > (iovlen + 2)); } ret = NXT_OK; - if (nxt_slow_path(nmount(iov, iovlen, 0) < 0)) { + if (nxt_slow_path(nmount(iov, iovlen, flags) < 0)) { nxt_alert(task, "nmount(%p, %d, 0) %s", iov, iovlen, errmsg); ret = NXT_ERROR; } diff --git a/src/nxt_fs.h b/src/nxt_fs.h index bbd7ab9f..ff589979 100644 --- a/src/nxt_fs.h +++ b/src/nxt_fs.h @@ -6,50 +6,33 @@ #define _NXT_FS_H_INCLUDED_ -#ifdef MS_BIND -#define NXT_MS_BIND MS_BIND -#else -#define NXT_MS_BIND 0 -#endif - -#ifdef MS_REC -#define NXT_MS_REC MS_BIND -#else -#define NXT_MS_REC 0 -#endif - -#ifdef MS_NOSUID -#define NXT_MS_NOSUID MS_NOSUID -#else -#define NXT_MS_NOSUID 0 -#endif - -#ifdef MS_NOEXEC -#define NXT_MS_NOEXEC MS_NOEXEC -#else -#define NXT_MS_NOEXEC 0 -#endif - -#ifdef MS_RELATIME -#define NXT_MS_RELATIME MS_RELATIME -#else -#define NXT_MS_RELATIME 0 -#endif - -#ifdef MS_NODEV -#define NXT_MS_NODEV MS_NODEV -#else -#define NXT_MS_NODEV 0 -#endif +typedef enum { + NXT_FS_UNKNOWN = 0, + NXT_FS_BIND, + NXT_FS_TMP, + NXT_FS_PROC, + NXT_FS_LAST, +} nxt_fs_type_t; + + +typedef enum { + NXT_FS_FLAGS_NOSUID = 1 << 0, + NXT_FS_FLAGS_NOEXEC = 1 << 1, + NXT_FS_FLAGS_NOTIME = 1 << 2, + NXT_FS_FLAGS_NODEV = 1 << 3, + NXT_FS_FLAGS_RDONLY = 1 << 4, +} nxt_fs_flags_t; typedef struct { - u_char *src; - u_char *dst; - u_char *fstype; - nxt_int_t flags; - u_char *data; - nxt_uint_t builtin; /* 1-bit */ + u_char *src; + u_char *dst; + nxt_fs_type_t type; + u_char *name; + nxt_fs_flags_t flags; + u_char *data; + nxt_uint_t builtin; /* 1-bit */ + nxt_uint_t deps; /* 1-bit */ } nxt_fs_mount_t; diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index dc23d7c4..dccbe56c 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -467,6 +467,7 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data) nxt_int_t ret; nxt_conn_t *c; nxt_h1proto_t *h1p; + nxt_socket_conf_t *skcf; nxt_http_request_t *r; nxt_socket_conf_joint_t *joint; @@ -503,11 +504,14 @@ nxt_h1p_conn_request_init(nxt_task_t *task, void *obj, void *data) joint->count++; r->conf = joint; + skcf = joint->socket_conf; if (c->local == NULL) { - c->local = joint->socket_conf->sockaddr; + c->local = skcf->sockaddr; } + h1p->parser.discard_unsafe_fields = skcf->discard_unsafe_fields; + nxt_h1p_conn_request_header_parse(task, c, h1p); return; } diff --git a/src/nxt_http.h b/src/nxt_http.h index 08181520..1418be95 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -7,6 +7,8 @@ #ifndef _NXT_HTTP_H_INCLUDED_ #define _NXT_HTTP_H_INCLUDED_ +#include <nxt_regex.h> + typedef enum { NXT_HTTP_UNSET = -1, @@ -168,6 +170,10 @@ struct nxt_http_request_s { void *req_rpc_data; +#if (NXT_HAVE_REGEX) + nxt_regex_match_t *regex_match; +#endif + nxt_http_peer_t *peer; nxt_buf_t *last; diff --git a/src/nxt_http_parse.c b/src/nxt_http_parse.c index 22004cc1..338b0a90 100644 --- a/src/nxt_http_parse.c +++ b/src/nxt_http_parse.c @@ -288,11 +288,13 @@ continue_target: case NXT_HTTP_TARGET_SPACE: rp->target_end = p; goto space_after_target; - +#if 0 case NXT_HTTP_TARGET_QUOTE_MARK: rp->quoted_target = 1; goto rest_of_target; - +#else + case NXT_HTTP_TARGET_QUOTE_MARK: +#endif case NXT_HTTP_TARGET_HASH: rp->complex_target = 1; goto rest_of_target; @@ -378,7 +380,7 @@ space_after_target: } } - rp->space_in_target = 1; + //rp->space_in_target = 1; if (rest) { goto rest_of_target; @@ -397,7 +399,7 @@ space_after_target: goto space_after_target; } - rp->space_in_target = 1; + //rp->space_in_target = 1; if (rest) { goto rest_of_target; @@ -432,7 +434,12 @@ space_after_target: *pos = p + 10; } - if (rp->complex_target != 0 || rp->quoted_target != 0) { + if (rp->complex_target != 0 +#if 0 + || rp->quoted_target != 0 +#endif + ) + { rc = nxt_http_parse_complex_target(rp); if (nxt_slow_path(rc != NXT_OK)) { @@ -518,11 +525,13 @@ nxt_http_parse_field_name(nxt_http_request_parse_t *rp, u_char **pos, static const u_char normal[256] nxt_aligned(64) = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0\0\0\0-\0\0" "0123456789\0\0\0\0\0\0" + /* \s ! " # $ % & ' ( ) * + , . / : ; < = > ? */ + "\0\1\0\1\1\1\1\1\0\0\1\1\0" "-" "\1\0" "0123456789" "\0\0\0\0\0\0" - /* These 64 bytes should reside in one cache line. */ - "\0abcdefghijklmnopqrstuvwxyz\0\0\0\0_" - "\0abcdefghijklmnopqrstuvwxyz\0\0\0\0\0" + /* @ [ \ ] ^ _ */ + "\0" "abcdefghijklmnopqrstuvwxyz" "\0\0\0\1\1" + /* ` { | } ~ */ + "\1" "abcdefghijklmnopqrstuvwxyz" "\0\1\0\1\0" "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" @@ -538,9 +547,14 @@ nxt_http_parse_field_name(nxt_http_request_parse_t *rp, u_char **pos, \ c = normal[ch]; \ \ - if (nxt_slow_path(c == '\0')) { \ - p = &(ch); \ - goto name_end; \ + if (nxt_slow_path(c <= '\1')) { \ + if (c == '\0') { \ + p = &(ch); \ + goto name_end; \ + } \ + \ + rp->skip_field = rp->discard_unsafe_fields; \ + c = ch; \ } \ \ hash = nxt_http_field_hash_char(hash, c); @@ -777,20 +791,25 @@ nxt_http_parse_field_end(nxt_http_request_parse_t *rp, u_char **pos, *pos = p + 1; if (rp->field_name.length != 0) { - field = nxt_list_add(rp->fields); + if (rp->skip_field) { + rp->skip_field = 0; - if (nxt_slow_path(field == NULL)) { - return NXT_ERROR; - } + } else { + field = nxt_list_add(rp->fields); - field->hash = nxt_http_field_hash_end(rp->field_hash); - field->skip = 0; - field->hopbyhop = 0; + if (nxt_slow_path(field == NULL)) { + return NXT_ERROR; + } - field->name_length = rp->field_name.length; - field->value_length = rp->field_value.length; - field->name = rp->field_name.start; - field->value = rp->field_value.start; + field->hash = nxt_http_field_hash_end(rp->field_hash); + field->skip = 0; + field->hopbyhop = 0; + + field->name_length = rp->field_name.length; + field->value_length = rp->field_value.length; + field->name = rp->field_name.start; + field->value = rp->field_value.start; + } rp->field_hash = NXT_HTTP_FIELD_HASH_INIT; @@ -1023,7 +1042,7 @@ nxt_http_parse_complex_target(nxt_http_request_parse_t *rp) break; case sw_quoted: - rp->quoted_target = 1; + //rp->quoted_target = 1; if (ch >= '0' && ch <= '9') { high = (u_char) (ch - '0'); diff --git a/src/nxt_http_parse.h b/src/nxt_http_parse.h index cbfc8433..3cd9bd15 100644 --- a/src/nxt_http_parse.h +++ b/src/nxt_http_parse.h @@ -55,15 +55,19 @@ struct nxt_http_request_parse_s { uint32_t field_hash; + uint8_t skip_field; /* 1 bit */ + uint8_t discard_unsafe_fields; /* 1 bit */ + /* target with "/." */ - uint8_t complex_target; /* 1 bit */ + uint8_t complex_target; /* 1 bit */ +#if 0 /* target with "%" */ - uint8_t quoted_target; /* 1 bit */ + uint8_t quoted_target; /* 1 bit */ /* target with " " */ - uint8_t space_in_target; /* 1 bit */ - + uint8_t space_in_target; /* 1 bit */ +#endif /* Preserve encoded '/' (%2F) and '%' (%25). */ - uint8_t encoded_slashes; /* 1 bit */ + uint8_t encoded_slashes; /* 1 bit */ }; diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 76fb3427..650c1a89 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -220,7 +220,7 @@ nxt_http_request_create(nxt_task_t *task) nxt_buf_t *last; nxt_http_request_t *r; - mp = nxt_mp_create(1024, 128, 256, 32); + mp = nxt_mp_create(4096, 128, 512, 32); if (nxt_slow_path(mp == NULL)) { return NULL; } diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index ae91076a..9aaa708e 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -8,6 +8,7 @@ #include <nxt_http.h> #include <nxt_sockaddr.h> #include <nxt_http_route_addr.h> +#include <nxt_regex.h> typedef enum { @@ -76,12 +77,20 @@ typedef struct { typedef struct { + union { + nxt_array_t *pattern_slices; +#if (NXT_HAVE_REGEX) + nxt_regex_t *regex; +#endif + } u; uint32_t min_length; - nxt_array_t *pattern_slices; uint8_t case_sensitive; /* 1 bit */ uint8_t negative; /* 1 bit */ uint8_t any; /* 1 bit */ +#if (NXT_HAVE_REGEX) + uint8_t regex; /* 1 bit */ +#endif } nxt_http_route_pattern_t; @@ -1060,24 +1069,24 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, nxt_str_t test, tmp; nxt_int_t ret; nxt_array_t *slices; +#if (NXT_HAVE_REGEX) + nxt_regex_t *re; + nxt_regex_err_t err; +#endif nxt_http_route_pattern_type_t type; - nxt_http_route_pattern_slice_t *slice; type = NXT_HTTP_ROUTE_PATTERN_EXACT; nxt_conf_get_string(cv, &test); - slices = nxt_array_create(mp, 1, sizeof(nxt_http_route_pattern_slice_t)); - if (nxt_slow_path(slices == NULL)) { - return NXT_ERROR; - } - - pattern->pattern_slices = slices; - + pattern->u.pattern_slices = NULL; pattern->negative = 0; pattern->any = 1; pattern->min_length = 0; +#if (NXT_HAVE_REGEX) + pattern->regex = 0; +#endif if (test.length != 0 && test.start[0] == '!') { test.start++; @@ -1087,6 +1096,41 @@ nxt_http_route_pattern_create(nxt_task_t *task, nxt_mp_t *mp, pattern->any = 0; } + if (test.length > 0 && test.start[0] == '~') { +#if (NXT_HAVE_REGEX) + test.start++; + test.length--; + + re = nxt_regex_compile(mp, &test, &err); + if (nxt_slow_path(re == NULL)) { + if (err.offset < test.length) { + nxt_alert(task, "nxt_regex_compile(%V) failed: %s at offset %d", + &test, err.msg, (int) err.offset); + return NXT_ERROR; + } + + nxt_alert(task, "nxt_regex_compile(%V) failed %s", &test, err.msg); + + return NXT_ERROR; + } + + pattern->u.regex = re; + pattern->regex = 1; + + return NXT_OK; + +#else + return NXT_ERROR; +#endif + } + + slices = nxt_array_create(mp, 1, sizeof(nxt_http_route_pattern_slice_t)); + if (nxt_slow_path(slices == NULL)) { + return NXT_ERROR; + } + + pattern->u.pattern_slices = slices; + if (test.length == 0) { slice = nxt_array_add(slices); if (nxt_slow_path(slice == NULL)) { @@ -1980,6 +2024,9 @@ nxt_http_route_header(nxt_http_request_t *r, nxt_http_route_rule_t *rule) } ret = nxt_http_route_test_rule(r, rule, f->value, f->value_length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } if (ret == 0) { return ret; @@ -2155,7 +2202,7 @@ static nxt_int_t nxt_http_route_test_argument(nxt_http_request_t *r, nxt_http_route_rule_t *rule, nxt_array_t *array) { - nxt_bool_t ret; + nxt_int_t ret; nxt_http_name_value_t *nv, *end; ret = 0; @@ -2171,6 +2218,10 @@ nxt_http_route_test_argument(nxt_http_request_t *r, { ret = nxt_http_route_test_rule(r, rule, nv->value, nv->value_length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + if (ret == 0) { break; } @@ -2189,7 +2240,7 @@ nxt_http_route_scheme(nxt_http_request_t *r, nxt_http_route_rule_t *rule) nxt_bool_t tls, https; nxt_http_route_pattern_slice_t *pattern_slice; - pattern_slice = rule->pattern[0].pattern_slices->elts; + pattern_slice = rule->pattern[0].u.pattern_slices->elts; https = (pattern_slice->length == nxt_length("https")); tls = (r->tls != NULL); @@ -2337,7 +2388,7 @@ static nxt_int_t nxt_http_route_test_cookie(nxt_http_request_t *r, nxt_http_route_rule_t *rule, nxt_array_t *array) { - nxt_bool_t ret; + nxt_int_t ret; nxt_http_name_value_t *nv, *end; ret = 0; @@ -2353,6 +2404,10 @@ nxt_http_route_test_cookie(nxt_http_request_t *r, { ret = nxt_http_route_test_rule(r, rule, nv->value, nv->value_length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + if (ret == 0) { break; } @@ -2378,6 +2433,9 @@ nxt_http_route_test_rule(nxt_http_request_t *r, nxt_http_route_rule_t *rule, while (pattern < end) { ret = nxt_http_route_pattern(r, pattern, start, length); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } /* nxt_http_route_pattern() returns either 1 or 0. */ ret ^= pattern->negative; @@ -2403,11 +2461,26 @@ nxt_http_route_pattern(nxt_http_request_t *r, nxt_http_route_pattern_t *pattern, nxt_array_t *pattern_slices; nxt_http_route_pattern_slice_t *pattern_slice; +#if (NXT_HAVE_REGEX) + if (pattern->regex) { + if (r->regex_match == NULL) { + r->regex_match = nxt_regex_match_create(r->mem_pool, 0); + if (nxt_slow_path(r->regex_match == NULL)) { + return NXT_ERROR; + } + } + + return nxt_regex_match(pattern->u.regex, start, length, r->regex_match); + } +#endif + if (length < pattern->min_length) { return 0; } - pattern_slices = pattern->pattern_slices; + nxt_assert(pattern->u.pattern_slices != NULL); + + pattern_slices = pattern->u.pattern_slices; pattern_slice = pattern_slices->elts; end = start + length; diff --git a/src/nxt_isolation.c b/src/nxt_isolation.c index ac7a37e8..1e6323bc 100644 --- a/src/nxt_isolation.c +++ b/src/nxt_isolation.c @@ -41,6 +41,8 @@ static nxt_int_t nxt_isolation_set_mounts(nxt_task_t *task, nxt_process_t *process, nxt_str_t *app_type); static nxt_int_t nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, nxt_array_t *syspaths); +static int nxt_cdecl nxt_isolation_mount_compare(const void *v1, + const void *v2); static void nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process); #if (NXT_HAVE_PIVOT_ROOT) && (NXT_HAVE_CLONE_NEWNS) @@ -85,15 +87,6 @@ nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, } #endif -#if (NXT_HAVE_ISOLATION_ROOTFS) - if (process->isolation.rootfs != NULL) { - ret = nxt_isolation_set_mounts(task, process, &app_conf->type); - if (nxt_slow_path(ret != NXT_OK)) { - return ret; - } - } -#endif - if (cap_setid) { ret = nxt_process_creds_set(task, process, &app_conf->user, &app_conf->group); @@ -124,6 +117,29 @@ nxt_isolation_main_prefork(nxt_task_t *task, nxt_process_t *process, } } +#if (NXT_HAVE_ISOLATION_ROOTFS) + if (process->isolation.rootfs != NULL) { + nxt_int_t has_mnt; + + ret = nxt_isolation_set_mounts(task, process, &app_conf->type); + if (nxt_slow_path(ret != NXT_OK)) { + return ret; + } + + has_mnt = 0; + +#if (NXT_HAVE_CLONE_NEWNS) + has_mnt = nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS); +#endif + + if (process->user_cred->uid == 0 && !has_mnt) { + nxt_log(task, NXT_LOG_WARN, + "setting user \"root\" with \"rootfs\" is unsafe without " + "\"mount\" namespace isolation"); + } + } +#endif + #if (NXT_HAVE_CLONE_NEWUSER) ret = nxt_isolation_vldt_creds(task, process); if (nxt_slow_path(ret != NXT_OK)) { @@ -468,10 +484,14 @@ nxt_isolation_set_automount(nxt_task_t *task, nxt_conf_value_t *isolation, static nxt_str_t automount_name = nxt_string("automount"); static nxt_str_t langdeps_name = nxt_string("language_deps"); + static nxt_str_t tmp_name = nxt_string("tmpfs"); + static nxt_str_t proc_name = nxt_string("procfs"); automount = &process->isolation.automount; automount->language_deps = 1; + automount->tmpfs = 1; + automount->procfs = 1; conf = nxt_conf_get_object_member(isolation, &automount_name, NULL); if (conf != NULL) { @@ -479,6 +499,16 @@ nxt_isolation_set_automount(nxt_task_t *task, nxt_conf_value_t *isolation, if (value != NULL) { automount->language_deps = nxt_conf_get_boolean(value); } + + value = nxt_conf_get_object_member(conf, &tmp_name, NULL); + if (value != NULL) { + automount->tmpfs = nxt_conf_get_boolean(value); + } + + value = nxt_conf_get_object_member(conf, &proc_name, NULL); + if (value != NULL) { + automount->procfs = nxt_conf_get_boolean(value); + } } return NXT_OK; @@ -560,39 +590,41 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, *p = '\0'; } - mnt = nxt_array_add(mounts); - if (nxt_slow_path(mnt == NULL)) { - return NXT_ERROR; - } + if (process->isolation.automount.tmpfs) { + mnt = nxt_array_add(mounts); + if (nxt_slow_path(mnt == NULL)) { + return NXT_ERROR; + } - mnt->src = (u_char *) "tmpfs"; - mnt->fstype = (u_char *) "tmpfs"; - mnt->flags = NXT_MS_NOSUID | NXT_MS_NODEV | NXT_MS_NOEXEC | NXT_MS_RELATIME; - mnt->data = (u_char *) "size=1m,mode=777"; - mnt->builtin = 1; + mnt->src = (u_char *) "tmpfs"; + mnt->name = (u_char *) "tmpfs"; + mnt->type = NXT_FS_TMP; + mnt->flags = (NXT_FS_FLAGS_NOSUID + | NXT_FS_FLAGS_NODEV + | NXT_FS_FLAGS_NOEXEC); + mnt->data = (u_char *) "size=1m,mode=777"; + mnt->builtin = 1; + mnt->deps = 0; + + mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/tmp") + 1); + if (nxt_slow_path(mnt->dst == NULL)) { + return NXT_ERROR; + } - mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/tmp") + 1); - if (nxt_slow_path(mnt->dst == NULL)) { - return NXT_ERROR; + p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); + p = nxt_cpymem(p, "/tmp", 4); + *p = '\0'; } - p = nxt_cpymem(mnt->dst, rootfs, rootfs_len); - p = nxt_cpymem(p, "/tmp", 4); - *p = '\0'; - -#if (NXT_HAVE_CLONE_NEWPID) && (NXT_HAVE_CLONE_NEWNS) - - if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWPID) - && nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS)) - { + if (process->isolation.automount.procfs) { mnt = nxt_array_add(mounts); if (nxt_slow_path(mnt == NULL)) { return NXT_ERROR; } - mnt->fstype = (u_char *) "proc"; - mnt->src = (u_char *) "proc"; - + mnt->name = (u_char *) "proc"; + mnt->type = NXT_FS_PROC; + mnt->src = (u_char *) "none"; mnt->dst = nxt_mp_nget(mp, rootfs_len + nxt_length("/proc") + 1); if (nxt_slow_path(mnt->dst == NULL)) { return NXT_ERROR; @@ -603,9 +635,13 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, *p = '\0'; mnt->data = (u_char *) ""; - mnt->flags = 0; + mnt->flags = NXT_FS_FLAGS_NOEXEC | NXT_FS_FLAGS_NOSUID; + mnt->builtin = 1; + mnt->deps = 0; } -#endif + + qsort(mounts->elts, mounts->nelts, sizeof(nxt_fs_mount_t), + nxt_isolation_mount_compare); process->isolation.mounts = mounts; @@ -613,14 +649,39 @@ nxt_isolation_set_lang_mounts(nxt_task_t *task, nxt_process_t *process, } +static int nxt_cdecl +nxt_isolation_mount_compare(const void *v1, const void *v2) +{ + const nxt_fs_mount_t *mnt1, *mnt2; + + mnt1 = v1; + mnt2 = v2; + + return nxt_strlen(mnt1->src) > nxt_strlen(mnt2->src); +} + + void nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) { - size_t i, n; + size_t n; nxt_array_t *mounts; + nxt_runtime_t *rt; nxt_fs_mount_t *mnt; nxt_process_automount_t *automount; + rt = task->thread->runtime; + + if (!rt->capabilities.setid) { + return; + } + +#if (NXT_HAVE_CLONE_NEWNS) + if (nxt_is_clone_flag_set(process->isolation.clone.flags, NEWNS)) { + return; + } +#endif + nxt_debug(task, "unmount all (%s)", process->name); automount = &process->isolation.automount; @@ -628,12 +689,14 @@ nxt_isolation_unmount_all(nxt_task_t *task, nxt_process_t *process) n = mounts->nelts; mnt = mounts->elts; - for (i = 0; i < n; i++) { - if (mnt[i].builtin && !automount->language_deps) { + while (n > 0) { + n--; + + if (mnt[n].deps && !automount->language_deps) { continue; } - nxt_fs_unmount(mnt[i].dst); + nxt_fs_unmount(mnt[n].dst); } } @@ -658,11 +721,11 @@ nxt_isolation_prepare_rootfs(nxt_task_t *task, nxt_process_t *process) for (i = 0; i < n; i++) { dst = mnt[i].dst; - if (mnt[i].builtin && !automount->language_deps) { + if (mnt[i].deps && !automount->language_deps) { continue; } - if (nxt_slow_path(nxt_memcmp(mnt[i].fstype, "bind", 4) == 0 + if (nxt_slow_path(mnt[i].type == NXT_FS_BIND && stat((const char *) mnt[i].src, &st) != 0)) { nxt_log(task, NXT_LOG_WARN, "host path not found: %s", mnt[i].src); diff --git a/src/nxt_java.c b/src/nxt_java.c index 1f8864bd..ac715c0b 100644 --- a/src/nxt_java.c +++ b/src/nxt_java.c @@ -36,6 +36,11 @@ static nxt_int_t nxt_java_start(nxt_task_t *task, static void nxt_java_request_handler(nxt_unit_request_info_t *req); static void nxt_java_websocket_handler(nxt_unit_websocket_frame_t *ws); static void nxt_java_close_handler(nxt_unit_request_info_t *req); +static int nxt_java_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_java_thread_func(void *main_ctx); +static int nxt_java_init_threads(nxt_java_app_conf_t *c); +static void nxt_java_join_threads(nxt_unit_ctx_t *ctx, + nxt_java_app_conf_t *c); static uint32_t compat[] = { NXT_VERNUM, NXT_DEBUG, @@ -43,6 +48,9 @@ static uint32_t compat[] = { char *nxt_java_modules; +static pthread_t *nxt_java_threads; +static pthread_attr_t *nxt_java_thread_attr; + #define NXT_STRING(x) _NXT_STRING(x) #define _NXT_STRING(x) #x @@ -59,8 +67,10 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { }; typedef struct { - JNIEnv *env; - jobject ctx; + JavaVM *jvm; + jobject cl; + jobject ctx; + nxt_java_app_conf_t *conf; } nxt_java_data_t; @@ -402,8 +412,10 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) goto env_failed; } - java_data.env = env; + java_data.jvm = jvm; + java_data.cl = cl; java_data.ctx = nxt_java_startContext(env, c->webapp, classpath); + java_data.conf = c; if ((*env)->ExceptionCheck(env)) { nxt_alert(task, "Unhandled exception in application start"); @@ -411,13 +423,20 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) return NXT_ERROR; } + rc = nxt_java_init_threads(c); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + return NXT_ERROR; + } + nxt_unit_default_init(task, &java_init); java_init.callbacks.request_handler = nxt_java_request_handler; java_init.callbacks.websocket_handler = nxt_java_websocket_handler; java_init.callbacks.close_handler = nxt_java_close_handler; + java_init.callbacks.ready_handler = nxt_java_ready_handler; java_init.request_data_size = sizeof(nxt_java_request_data_t); java_init.data = &java_data; + java_init.ctx_data = env; java_init.shm_limit = app_conf->shm_limit; ctx = nxt_unit_init(&java_init); @@ -427,9 +446,8 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) } rc = nxt_unit_run(ctx); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - /* TODO report error */ - } + + nxt_java_join_threads(ctx, c); nxt_java_stopContext(env, java_data.ctx); @@ -441,7 +459,7 @@ nxt_java_start(nxt_task_t *task, nxt_process_data_t *data) (*jvm)->DestroyJavaVM(jvm); - exit(0); + exit(rc); return NXT_OK; @@ -464,7 +482,7 @@ nxt_java_request_handler(nxt_unit_request_info_t *req) nxt_java_request_data_t *data; java_data = req->unit->data; - env = java_data->env; + env = req->ctx->data; data = req->data; jreq = nxt_java_newRequest(env, java_data->ctx, req); @@ -543,11 +561,9 @@ nxt_java_websocket_handler(nxt_unit_websocket_frame_t *ws) void *b; JNIEnv *env; jobject jbuf; - nxt_java_data_t *java_data; nxt_java_request_data_t *data; - java_data = ws->req->unit->data; - env = java_data->env; + env = ws->req->ctx->data; data = ws->req->data; b = malloc(ws->payload_len); @@ -578,11 +594,9 @@ static void nxt_java_close_handler(nxt_unit_request_info_t *req) { JNIEnv *env; - nxt_java_data_t *java_data; nxt_java_request_data_t *data; - java_data = req->unit->data; - env = java_data->env; + env = req->ctx->data; data = req->data; nxt_java_Request_close(env, data->jreq); @@ -593,3 +607,160 @@ nxt_java_close_handler(nxt_unit_request_info_t *req) nxt_unit_request_done(req, NXT_UNIT_OK); } + +static int +nxt_java_ready_handler(nxt_unit_ctx_t *ctx) +{ + int res; + uint32_t i; + nxt_java_data_t *java_data; + nxt_java_app_conf_t *c; + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + java_data = ctx->unit->data; + c = java_data->conf; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + res = pthread_create(&nxt_java_threads[i], nxt_java_thread_attr, + nxt_java_thread_func, ctx); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d create failed: %s (%d)", + (int) (i + 1), strerror(res), res); + + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void * +nxt_java_thread_func(void *data) +{ + int rc; + JavaVM *jvm; + JNIEnv *env; + nxt_unit_ctx_t *main_ctx, *ctx; + nxt_java_data_t *java_data; + + main_ctx = data; + + nxt_unit_debug(main_ctx, "worker thread start"); + + java_data = main_ctx->unit->data; + jvm = java_data->jvm; + + rc = (*jvm)->AttachCurrentThread(jvm, (void **) &env, NULL); + if (rc != JNI_OK) { + nxt_unit_alert(main_ctx, "failed to attach Java VM: %d", (int) rc); + return NULL; + } + + nxt_java_setContextClassLoader(env, java_data->cl); + + ctx = nxt_unit_ctx_alloc(main_ctx, env); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + (void) nxt_unit_run(ctx); + + nxt_unit_done(ctx); + +fail: + + (*jvm)->DetachCurrentThread(jvm); + + nxt_unit_debug(NULL, "worker thread end"); + + return NULL; +} + + +static int +nxt_java_init_threads(nxt_java_app_conf_t *c) +{ + int res; + static pthread_attr_t attr; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + if (c->thread_stack_size > 0) { + res = pthread_attr_init(&attr); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr init failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + res = pthread_attr_setstacksize(&attr, c->thread_stack_size); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr set stack size failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + nxt_java_thread_attr = &attr; + } + + nxt_java_threads = nxt_unit_malloc(NULL, + sizeof(pthread_t) * (c->threads - 1)); + if (nxt_slow_path(nxt_java_threads == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate thread id array"); + + return NXT_UNIT_ERROR; + } + + memset(nxt_java_threads, 0, sizeof(pthread_t) * (c->threads - 1)); + + return NXT_UNIT_OK; +} + + +static void +nxt_java_join_threads(nxt_unit_ctx_t *ctx, nxt_java_app_conf_t *c) +{ + int res; + uint32_t i; + + if (nxt_java_threads == NULL) { + return; + } + + for (i = 0; i < c->threads - 1; i++) { + if ((uintptr_t) nxt_java_threads[i] == 0) { + continue; + } + + res = pthread_join(nxt_java_threads[i], NULL); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d joined", (int) i); + + } else { + nxt_unit_alert(ctx, "thread #%d join failed: %s (%d)", + (int) i, strerror(res), res); + } + } + + nxt_unit_free(ctx, nxt_java_threads); +} + + diff --git a/src/nxt_lib.c b/src/nxt_lib.c index 1634a2b8..aba07dda 100644 --- a/src/nxt_lib.c +++ b/src/nxt_lib.c @@ -91,9 +91,12 @@ nxt_lib_start(const char *app, char **argv, char ***envp) #elif (NXT_HPUX) n = mpctl(MPC_GETNUMSPUS, NULL, NULL); +#else + n = 0; + #endif - nxt_debug(&nxt_main_task, "ncpu: %ui", n); + nxt_debug(&nxt_main_task, "ncpu: %d", n); if (n > 1) { nxt_ncpu = n; diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index d2edab1d..0cde435b 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -197,6 +197,24 @@ static nxt_conf_map_t nxt_python_app_conf[] = { NXT_CONF_MAP_CSTRZ, offsetof(nxt_common_app_conf_t, u.python.callable), }, + + { + nxt_string("protocol"), + NXT_CONF_MAP_STR, + offsetof(nxt_common_app_conf_t, u.python.protocol), + }, + + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.python.threads), + }, + + { + nxt_string("thread_stack_size"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.python.thread_stack_size), + }, }; @@ -221,6 +239,18 @@ static nxt_conf_map_t nxt_perl_app_conf[] = { NXT_CONF_MAP_CSTRZ, offsetof(nxt_common_app_conf_t, u.perl.script), }, + + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.perl.threads), + }, + + { + nxt_string("thread_stack_size"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.perl.thread_stack_size), + }, }; @@ -230,6 +260,11 @@ static nxt_conf_map_t nxt_ruby_app_conf[] = { NXT_CONF_MAP_STR, offsetof(nxt_common_app_conf_t, u.ruby.script), }, + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.ruby.threads), + }, }; @@ -254,6 +289,16 @@ static nxt_conf_map_t nxt_java_app_conf[] = { NXT_CONF_MAP_CSTRZ, offsetof(nxt_common_app_conf_t, u.java.unit_jars), }, + { + nxt_string("threads"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.java.threads), + }, + { + nxt_string("thread_stack_size"), + NXT_CONF_MAP_INT32, + offsetof(nxt_common_app_conf_t, u.java.thread_stack_size), + }, }; @@ -1163,9 +1208,14 @@ static nxt_conf_map_t nxt_app_lang_mounts_map[] = { offsetof(nxt_fs_mount_t, dst), }, { - nxt_string("fstype"), + nxt_string("name"), NXT_CONF_MAP_CSTRZ, - offsetof(nxt_fs_mount_t, fstype), + offsetof(nxt_fs_mount_t, name), + }, + { + nxt_string("type"), + NXT_CONF_MAP_INT, + offsetof(nxt_fs_mount_t, type), }, { nxt_string("flags"), @@ -1297,6 +1347,7 @@ nxt_main_port_modules_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) } mnt->builtin = 1; + mnt->deps = 1; ret = nxt_conf_map_object(rt->mem_pool, value, nxt_app_lang_mounts_map, diff --git a/src/nxt_pcre.c b/src/nxt_pcre.c new file mode 100644 index 00000000..737fc7cf --- /dev/null +++ b/src/nxt_pcre.c @@ -0,0 +1,135 @@ +/* + * Copyright (C) Axel Duch + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_regex.h> +#include <pcre.h> + + +struct nxt_regex_s { + pcre *code; + pcre_extra *extra; + nxt_str_t pattern; +}; + +struct nxt_regex_match_s { + int ovecsize; + int ovec[]; +}; + + +static void *nxt_pcre_malloc(size_t size); +static void nxt_pcre_free(void *p); + +static nxt_mp_t *nxt_pcre_mp; + + +nxt_regex_t * +nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) +{ + int erroffset; + char *pattern; + void *saved_malloc, *saved_free; + nxt_regex_t *re; + + err->offset = source->length; + + re = nxt_mp_get(mp, sizeof(nxt_regex_t) + source->length + 1); + if (nxt_slow_path(re == NULL)) { + err->msg = "memory allocation failed"; + return NULL; + } + + pattern = nxt_pointer_to(re, sizeof(nxt_regex_t)); + + nxt_memcpy(pattern, source->start, source->length); + pattern[source->length] = '\0'; + + re->pattern.length = source->length; + re->pattern.start = (u_char *) pattern; + + saved_malloc = pcre_malloc; + saved_free = pcre_free; + + pcre_malloc = nxt_pcre_malloc; + pcre_free = nxt_pcre_free; + nxt_pcre_mp = mp; + + re->code = pcre_compile(pattern, 0, &err->msg, &erroffset, NULL); + if (nxt_fast_path(re->code != NULL)) { +#if 0 + re->extra = pcre_study(re->code, PCRE_STUDY_JIT_COMPILE, &err->msg); + if (nxt_slow_path(re->extra == NULL && err->msg != NULL)) { + nxt_log_warn(thr->log, "pcre_study(%V) failed: %s", source, err->msg); + } +#else + re->extra = NULL; +#endif + + } else { + err->offset = erroffset; + re = NULL; + } + + pcre_malloc = saved_malloc; + pcre_free = saved_free; + + return re; +} + + +static void* +nxt_pcre_malloc(size_t size) +{ + if (nxt_slow_path(nxt_pcre_mp == NULL)) { + nxt_thread_log_alert("pcre_malloc(%uz) called without memory pool", + size); + return NULL; + } + + nxt_thread_log_debug("pcre_malloc(%uz), pool %p", size, nxt_pcre_mp); + + return nxt_mp_get(nxt_pcre_mp, size); +} + + +static void +nxt_pcre_free(void *p) +{ +} + + +nxt_regex_match_t * +nxt_regex_match_create(nxt_mp_t *mp, size_t size) +{ + nxt_regex_match_t *match; + + match = nxt_mp_get(mp, sizeof(nxt_regex_match_t) + sizeof(int) * size); + if (nxt_fast_path(match != NULL)) { + match->ovecsize = size; + } + + return match; +} + + +nxt_int_t +nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, + nxt_regex_match_t *match) +{ + int ret; + + ret = pcre_exec(re->code, re->extra, (const char *) subject, length, 0, 0, + match->ovec, match->ovecsize); + if (nxt_slow_path(ret < PCRE_ERROR_NOMATCH)) { + nxt_thread_log_error(NXT_LOG_ERR, + "pcre_exec() failed: %d on \"%*s\" using \"%V\"", + ret, length, subject, &re->pattern); + + return NXT_ERROR; + } + + return (ret != PCRE_ERROR_NOMATCH); +} diff --git a/src/nxt_pcre2.c b/src/nxt_pcre2.c new file mode 100644 index 00000000..22c4d2d4 --- /dev/null +++ b/src/nxt_pcre2.c @@ -0,0 +1,161 @@ +/* + * Copyright (C) Axel Duch + * Copyright (C) NGINX, Inc. + */ + +#include <nxt_main.h> +#include <nxt_regex.h> + +#define PCRE2_CODE_UNIT_WIDTH 8 +#include <pcre2.h> + + +static void *nxt_pcre2_malloc(PCRE2_SIZE size, void *memory_data); +static void nxt_pcre2_free(void *p, void *memory_data); + + +struct nxt_regex_s { + pcre2_code *code; + nxt_str_t pattern; +}; + + +nxt_regex_t * +nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, nxt_regex_err_t *err) +{ + int errcode; + nxt_int_t ret; + PCRE2_SIZE erroffset; + nxt_regex_t *re; + pcre2_general_context *general_ctx; + pcre2_compile_context *compile_ctx; + + static const u_char alloc_error[] = "memory allocation failed"; + + general_ctx = pcre2_general_context_create(nxt_pcre2_malloc, + nxt_pcre2_free, mp); + if (nxt_slow_path(general_ctx == NULL)) { + goto alloc_fail; + } + + compile_ctx = pcre2_compile_context_create(general_ctx); + if (nxt_slow_path(compile_ctx == NULL)) { + goto alloc_fail; + } + + re = nxt_mp_get(mp, sizeof(nxt_regex_t)); + if (nxt_slow_path(re == NULL)) { + goto alloc_fail; + } + + if (nxt_slow_path(nxt_str_dup(mp, &re->pattern, source) == NULL)) { + goto alloc_fail; + } + + re->code = pcre2_compile((PCRE2_SPTR) source->start, source->length, 0, + &errcode, &erroffset, compile_ctx); + if (nxt_slow_path(re->code == NULL)) { + err->offset = erroffset; + + ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, + ERR_BUF_SIZE); + if (ret < 0) { + (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, + "compilation failed with unknown " + "error code: %d%Z", errcode); + } + + return NULL; + } + +#if 0 + errcode = pcre2_jit_compile(re, PCRE2_JIT_COMPLETE); + if (nxt_slow_path(errcode != 0 && errcode != PCRE2_ERROR_JIT_BADOPTION)) { + ret = pcre2_get_error_message(errcode, (PCRE2_UCHAR *) err->msg, + ERR_BUF_SIZE); + if (ret < 0) { + (void) nxt_sprintf(err->msg, err->msg + ERR_BUF_SIZE, + "JIT compilation failed with unknown " + "error code: %d%Z", errcode); + } + + return NULL; + } +#endif + + return re; + +alloc_fail: + + err->offset = source->length; + nxt_memcpy(err->msg, alloc_error, sizeof(alloc_error)); + + return NULL; +} + + +static void * +nxt_pcre2_malloc(PCRE2_SIZE size, void *mp) +{ + return nxt_mp_get(mp, size); +} + + +static void +nxt_pcre2_free(void *p, void *mp) +{ +} + + +nxt_regex_match_t * +nxt_regex_match_create(nxt_mp_t *mp, size_t size) +{ + nxt_regex_match_t *match; + pcre2_general_context *ctx; + + ctx = pcre2_general_context_create(nxt_pcre2_malloc, nxt_pcre2_free, mp); + if (nxt_slow_path(ctx == NULL)) { + nxt_thread_log_alert("pcre2_general_context_create() failed"); + return NULL; + } + + match = pcre2_match_data_create(size, ctx); + if (nxt_slow_path(match == NULL)) { + nxt_thread_log_alert("pcre2_match_data_create(%uz) failed", size); + return NULL; + } + + return match; +} + + +nxt_int_t +nxt_regex_match(nxt_regex_t *re, u_char *subject, size_t length, + nxt_regex_match_t *match) +{ + nxt_int_t ret; + PCRE2_UCHAR errptr[ERR_BUF_SIZE]; + + ret = pcre2_match(re->code, (PCRE2_SPTR) subject, length, 0, 0, match, + NULL); + + if (nxt_slow_path(ret < PCRE2_ERROR_NOMATCH)) { + + if (pcre2_get_error_message(ret, errptr, ERR_BUF_SIZE) < 0) { + nxt_thread_log_error(NXT_LOG_ERR, + "pcre2_match() failed: %d on \"%*s\" " + "using \"%V\"", ret, subject, length, subject, + &re->pattern); + + } else { + nxt_thread_log_error(NXT_LOG_ERR, + "pcre2_match() failed: %s (%d) on \"%*s\" " + "using \"%V\"", errptr, ret, length, subject, + &re->pattern); + } + + return NXT_ERROR; + } + + return (ret != PCRE2_ERROR_NOMATCH); +} diff --git a/src/nxt_php_sapi.c b/src/nxt_php_sapi.c index 234ceef8..d2fbdd27 100644 --- a/src/nxt_php_sapi.c +++ b/src/nxt_php_sapi.c @@ -8,6 +8,7 @@ #include "SAPI.h" #include "php_main.h" #include "php_variables.h" +#include "ext/standard/php_standard.h" #include <nxt_main.h> #include <nxt_router.h> @@ -137,16 +138,38 @@ static int nxt_php_read_post(char *buffer, uint count_bytes TSRMLS_DC); #endif +#ifdef NXT_PHP7 +#if PHP_VERSION_ID < 70200 +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_fastcgi_finish_request, 0, 0, + _IS_BOOL, NULL, 0) +#else +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_fastcgi_finish_request, 0, 0, + _IS_BOOL, 0) +#endif +#else /* PHP5 */ +ZEND_BEGIN_ARG_INFO_EX(arginfo_fastcgi_finish_request, 0, 0, 0) +#endif +ZEND_END_ARG_INFO() + +ZEND_FUNCTION(fastcgi_finish_request); + PHP_MINIT_FUNCTION(nxt_php_ext); ZEND_NAMED_FUNCTION(nxt_php_chdir); + +static const zend_function_entry nxt_php_ext_functions[] = { + ZEND_FE(fastcgi_finish_request, arginfo_fastcgi_finish_request) + ZEND_FE_END +}; + + zif_handler nxt_php_chdir_handler; static zend_module_entry nxt_php_unit_module = { STANDARD_MODULE_HEADER, "unit", - NULL, /* function table */ + nxt_php_ext_functions, /* function table */ PHP_MINIT(nxt_php_ext), /* initialization */ NULL, /* shutdown */ NULL, /* request initialization */ @@ -186,6 +209,55 @@ ZEND_NAMED_FUNCTION(nxt_php_chdir) } +PHP_FUNCTION(fastcgi_finish_request) +{ + nxt_php_run_ctx_t *ctx; + + if (nxt_slow_path(zend_parse_parameters_none() == FAILURE)) { +#ifdef NXT_PHP8 + RETURN_THROWS(); +#else + return; +#endif + } + + ctx = SG(server_context); + + if (nxt_slow_path(ctx->req == NULL)) { + RETURN_FALSE; + } + +#ifdef NXT_PHP7 + php_output_end_all(); + php_header(); +#else +#ifdef PHP_OUTPUT_NEWAPI + php_output_end_all(TSRMLS_C); +#else + php_end_ob_buffers(1 TSRMLS_CC); +#endif + + php_header(TSRMLS_C); +#endif + + nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + ctx->req = NULL; + + PG(connection_status) = PHP_CONNECTION_ABORTED; +#ifdef NXT_PHP7 + php_output_set_status(PHP_OUTPUT_DISABLED); +#else +#ifdef PHP_OUTPUT_NEWAPI + php_output_set_status(PHP_OUTPUT_DISABLED TSRMLS_CC); +#else + php_output_set_status(0 TSRMLS_CC); +#endif +#endif + + RETURN_TRUE; +} + + static sapi_module_struct nxt_php_sapi_module = { (char *) "cli-server", @@ -934,6 +1006,9 @@ nxt_php_dynamic_request(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) static void nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) { +#if (PHP_VERSION_ID < 50600) + void *read_post; +#endif nxt_unit_field_t *f; zend_file_handle file_handle; @@ -990,9 +1065,23 @@ nxt_php_execute(nxt_php_run_ctx_t *ctx, nxt_unit_request_t *r) php_execute_script(&file_handle TSRMLS_CC); + /* Prevention of consuming possible unread request body. */ +#if (PHP_VERSION_ID < 50600) + read_post = sapi_module.read_post; + sapi_module.read_post = NULL; +#else + SG(post_read) = 1; +#endif + php_request_shutdown(NULL); - nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + if (ctx->req != NULL) { + nxt_unit_request_done(ctx->req, NXT_UNIT_OK); + } + +#if (PHP_VERSION_ID < 50600) + sapi_module.read_post = read_post; +#endif } diff --git a/src/nxt_port.h b/src/nxt_port.h index 3ac8c735..5ece3bfa 100644 --- a/src/nxt_port.h +++ b/src/nxt_port.h @@ -26,6 +26,7 @@ struct nxt_port_handlers_s { nxt_port_handler_t change_file; nxt_port_handler_t new_port; nxt_port_handler_t get_port; + nxt_port_handler_t port_ack; nxt_port_handler_t mmap; nxt_port_handler_t get_mmap; @@ -84,6 +85,7 @@ typedef enum { _NXT_PORT_MSG_CHANGE_FILE = nxt_port_handler_idx(change_file), _NXT_PORT_MSG_NEW_PORT = nxt_port_handler_idx(new_port), _NXT_PORT_MSG_GET_PORT = nxt_port_handler_idx(get_port), + _NXT_PORT_MSG_PORT_ACK = nxt_port_handler_idx(port_ack), _NXT_PORT_MSG_MMAP = nxt_port_handler_idx(mmap), _NXT_PORT_MSG_GET_MMAP = nxt_port_handler_idx(get_mmap), @@ -120,6 +122,7 @@ typedef enum { NXT_PORT_MSG_CHANGE_FILE = nxt_msg_last(_NXT_PORT_MSG_CHANGE_FILE), NXT_PORT_MSG_NEW_PORT = nxt_msg_last(_NXT_PORT_MSG_NEW_PORT), NXT_PORT_MSG_GET_PORT = nxt_msg_last(_NXT_PORT_MSG_GET_PORT), + NXT_PORT_MSG_PORT_ACK = nxt_msg_last(_NXT_PORT_MSG_PORT_ACK), NXT_PORT_MSG_MMAP = nxt_msg_last(_NXT_PORT_MSG_MMAP) | NXT_PORT_MSG_SYNC, NXT_PORT_MSG_GET_MMAP = nxt_msg_last(_NXT_PORT_MSG_GET_MMAP), diff --git a/src/nxt_port_memory.c b/src/nxt_port_memory.c index 1e01629e..ae9f079c 100644 --- a/src/nxt_port_memory.c +++ b/src/nxt_port_memory.c @@ -17,6 +17,10 @@ #include <nxt_port_memory_int.h> +static void nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, + void *data); + + nxt_inline void nxt_port_mmap_handler_use(nxt_port_mmap_handler_t *mmap_handler, int i) { @@ -112,7 +116,6 @@ nxt_port_mmap_buf_completion(nxt_task_t *task, void *obj, void *data) u_char *p; nxt_mp_t *mp; nxt_buf_t *b, *next; - nxt_port_t *port; nxt_process_t *process; nxt_chunk_id_t c; nxt_port_mmap_header_t *hdr; @@ -171,14 +174,7 @@ complete_buf: { process = nxt_runtime_process_find(task->thread->runtime, hdr->src_pid); - if (process != NULL && !nxt_queue_is_empty(&process->ports)) { - port = nxt_process_port_first(process); - - if (port->type == NXT_PROCESS_APP) { - (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_SHM_ACK, - -1, 0, 0, NULL); - } - } + nxt_process_broadcast_shm_ack(task, process); } release_buf: @@ -976,3 +972,35 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b) return m; } + + +void +nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process) +{ + nxt_port_t *port; + + if (nxt_slow_path(process == NULL || nxt_queue_is_empty(&process->ports))) + { + return; + } + + port = nxt_process_port_first(process); + + if (port->type == NXT_PROCESS_APP) { + nxt_port_post(task, port, nxt_port_broadcast_shm_ack, process); + } +} + + +static void +nxt_port_broadcast_shm_ack(nxt_task_t *task, nxt_port_t *port, void *data) +{ + nxt_process_t *process; + + process = data; + + nxt_queue_each(port, &process->ports, nxt_port_t, link) { + (void) nxt_port_socket_write(task, port, NXT_PORT_MSG_SHM_ACK, + -1, 0, 0, NULL); + } nxt_queue_loop; +} diff --git a/src/nxt_port_memory.h b/src/nxt_port_memory.h index 8e71af3d..a2cdf5dd 100644 --- a/src/nxt_port_memory.h +++ b/src/nxt_port_memory.h @@ -71,4 +71,6 @@ nxt_port_mmap_get_method(nxt_task_t *task, nxt_port_t *port, nxt_buf_t *b); nxt_int_t nxt_shm_open(nxt_task_t *task, size_t size); +void nxt_process_broadcast_shm_ack(nxt_task_t *task, nxt_process_t *process); + #endif /* _NXT_PORT_MEMORY_H_INCLUDED_ */ diff --git a/src/nxt_process.c b/src/nxt_process.c index 9be7974f..87419313 100644 --- a/src/nxt_process.c +++ b/src/nxt_process.c @@ -248,8 +248,6 @@ nxt_process_setup(nxt_task_t *task, nxt_process_t *process) port = nxt_process_port_first(process); - nxt_port_write_close(port); - nxt_port_enable(task, port, init->port_handlers); ret = init->setup(task, process); @@ -272,6 +270,9 @@ nxt_process_setup(nxt_task_t *task, nxt_process_t *process) } ret = init->start(task, &process->data); + + nxt_port_write_close(port); + break; default: diff --git a/src/nxt_process.h b/src/nxt_process.h index d9b4dff1..7afb8803 100644 --- a/src/nxt_process.h +++ b/src/nxt_process.h @@ -74,7 +74,9 @@ typedef struct { typedef struct { - uint8_t language_deps; /* 1-byte */ + uint8_t language_deps; /* 1-bit */ + uint8_t tmpfs; /* 1-bit */ + uint8_t procfs; /* 1-bit */ } nxt_process_automount_t; diff --git a/src/nxt_regex.h b/src/nxt_regex.h new file mode 100644 index 00000000..a24c50f8 --- /dev/null +++ b/src/nxt_regex.h @@ -0,0 +1,41 @@ + +/* + * Copyright (C) Axel Duch + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_REGEX_H_INCLUDED_ +#define _NXT_REGEX_H_INCLUDED_ + +#if (NXT_HAVE_REGEX) + +typedef struct nxt_regex_s nxt_regex_t; + + #if (NXT_HAVE_PCRE2) +typedef void nxt_regex_match_t; +#else +typedef struct nxt_regex_match_s nxt_regex_match_t; +#endif + +typedef struct { + size_t offset; + +#if (NXT_HAVE_PCRE2) +#define ERR_BUF_SIZE 256 + u_char msg[ERR_BUF_SIZE]; +#else + const char *msg; +#endif +} nxt_regex_err_t; + + +NXT_EXPORT void nxt_regex_init(void); +NXT_EXPORT nxt_regex_t *nxt_regex_compile(nxt_mp_t *mp, nxt_str_t *source, + nxt_regex_err_t *err); +NXT_EXPORT nxt_regex_match_t *nxt_regex_match_create(nxt_mp_t *mp, size_t size); +NXT_EXPORT nxt_int_t nxt_regex_match(nxt_regex_t *re, u_char *subject, + size_t length, nxt_regex_match_t *match); + +#endif /* NXT_HAVE_REGEX */ + +#endif /* _NXT_REGEX_H_INCLUDED_ */ diff --git a/src/nxt_router.c b/src/nxt_router.c index a3218047..9dd5c30e 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -688,6 +688,8 @@ nxt_router_new_port_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) port->app = app; port->main_app_port = main_app_port; + + nxt_port_socket_write(task, port, NXT_PORT_MSG_PORT_ACK, -1, 0, 0, NULL); } @@ -1260,6 +1262,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = { NXT_CONF_MAP_STR, offsetof(nxt_socket_conf_t, body_temp_path), }, + + { + nxt_string("discard_unsafe_fields"), + NXT_CONF_MAP_INT8, + offsetof(nxt_socket_conf_t, discard_unsafe_fields), + }, }; @@ -1647,6 +1655,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->header_buffer_size = 2048; skcf->large_header_buffer_size = 8192; skcf->large_header_buffers = 4; + skcf->discard_unsafe_fields = 1; skcf->body_buffer_size = 16 * 1024; skcf->max_body_size = 8 * 1024 * 1024; skcf->proxy_header_buffer_size = 64 * 1024; @@ -3722,6 +3731,7 @@ static void nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, void *data) { + size_t b_size, count; nxt_int_t ret; nxt_app_t *app; nxt_buf_t *b, *next; @@ -3787,15 +3797,20 @@ nxt_router_response_ready_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg, nxt_http_request_send_body(task, r, NULL); } else { - size_t b_size = nxt_buf_mem_used_size(&b->mem); + b_size = nxt_buf_is_mem(b) ? nxt_buf_mem_used_size(&b->mem) : 0; - if (nxt_slow_path(b_size < sizeof(*resp))) { + if (nxt_slow_path(b_size < sizeof(nxt_unit_response_t))) { + nxt_alert(task, "response buffer too small: %z", b_size); goto fail; } resp = (void *) b->mem.pos; - if (nxt_slow_path(b_size < sizeof(*resp) - + resp->fields_count * sizeof(nxt_unit_field_t))) { + count = (b_size - sizeof(nxt_unit_response_t)) + / sizeof(nxt_unit_field_t); + + if (nxt_slow_path(count < resp->fields_count)) { + nxt_alert(task, "response buffer too small for fields count: %D", + resp->fields_count); goto fail; } @@ -3904,6 +3919,7 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, { int res; nxt_app_t *app; + nxt_buf_t *b; nxt_bool_t start_process, unlinked; nxt_port_t *app_port, *main_app_port, *idle_port; nxt_queue_link_t *idle_lnk; @@ -4001,16 +4017,25 @@ nxt_router_req_headers_ack_handler(nxt_task_t *task, req_rpc_data->app_port = app_port; - if (req_rpc_data->msg_info.body_fd != -1) { + b = req_rpc_data->msg_info.buf; + + if (b != NULL) { + /* First buffer is already sent. Start from second. */ + b = b->next; + } + + if (req_rpc_data->msg_info.body_fd != -1 || b != NULL) { nxt_debug(task, "stream #%uD: send body fd %d", req_rpc_data->stream, req_rpc_data->msg_info.body_fd); - lseek(req_rpc_data->msg_info.body_fd, 0, SEEK_SET); + if (req_rpc_data->msg_info.body_fd != -1) { + lseek(req_rpc_data->msg_info.body_fd, 0, SEEK_SET); + } res = nxt_port_socket_write(task, app_port, NXT_PORT_MSG_REQ_BODY, req_rpc_data->msg_info.body_fd, req_rpc_data->stream, - task->thread->engine->port->id, NULL); + task->thread->engine->port->id, b); if (nxt_slow_path(res != NXT_OK)) { nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); @@ -4667,6 +4692,25 @@ nxt_router_free_app(nxt_task_t *task, void *obj, void *data) nxt_port_use(task, port, -1); } + nxt_thread_mutex_lock(&app->mutex); + + for ( ;; ) { + port = nxt_port_hash_retrieve(&app->port_hash); + if (port == NULL) { + break; + } + + app->port_hash_count--; + + port->app = NULL; + + nxt_port_close(task, port); + + nxt_port_use(task, port, -1); + } + + nxt_thread_mutex_unlock(&app->mutex); + nxt_assert(app->processes == 0); nxt_assert(app->active_requests == 0); nxt_assert(app->port_hash_count == 0); @@ -5364,8 +5408,7 @@ nxt_router_oosm_handler(nxt_task_t *task, nxt_port_recv_msg_t *msg) nxt_thread_mutex_unlock(&process->incoming.mutex); if (ack) { - (void) nxt_port_socket_write(task, msg->port, NXT_PORT_MSG_SHM_ACK, - -1, 0, 0, NULL); + nxt_process_broadcast_shm_ack(task, process); } } diff --git a/src/nxt_router.h b/src/nxt_router.h index 512f1810..5804840f 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -191,6 +191,8 @@ typedef struct { nxt_str_t body_temp_path; + uint8_t discard_unsafe_fields; /* 1 bit */ + #if (NXT_TLS) nxt_tls_conf_t *tls; #endif diff --git a/src/nxt_runtime.c b/src/nxt_runtime.c index 44970b34..d9d986da 100644 --- a/src/nxt_runtime.c +++ b/src/nxt_runtime.c @@ -10,6 +10,7 @@ #include <nxt_port.h> #include <nxt_main_process.h> #include <nxt_router.h> +#include <nxt_regex.h> static nxt_int_t nxt_runtime_inherited_listen_sockets(nxt_task_t *task, @@ -702,9 +703,6 @@ nxt_runtime_thread_pool_destroy(nxt_task_t *task, nxt_runtime_t *rt, static void nxt_runtime_thread_pool_init(void) { -#if (NXT_REGEX) - nxt_regex_init(0); -#endif } diff --git a/src/nxt_unit.c b/src/nxt_unit.c index f75d61bc..097f50d6 100644 --- a/src/nxt_unit.c +++ b/src/nxt_unit.c @@ -54,11 +54,13 @@ static int nxt_unit_read_env(nxt_unit_port_t *ready_port, int *log_fd, uint32_t *stream, uint32_t *shm_limit); static int nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd); -static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf); +static int nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, + nxt_unit_request_info_t **preq); static int nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); +static int nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx); static int nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, - nxt_unit_recv_msg_t *recv_msg); + nxt_unit_recv_msg_t *recv_msg, nxt_unit_request_info_t **preq); static int nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg); static int nxt_unit_request_check_response_port(nxt_unit_request_info_t *req, @@ -106,6 +108,8 @@ static int nxt_unit_get_outgoing_buf(nxt_unit_ctx_t *ctx, uint32_t min_size, nxt_unit_mmap_buf_t *mmap_buf, char *local_buf); static int nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd); +static void nxt_unit_awake_ctx(nxt_unit_ctx_t *ctx, + nxt_unit_ctx_impl_t *ctx_impl); static void nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps); nxt_inline void nxt_unit_process_use(nxt_unit_process_t *process); nxt_inline void nxt_unit_process_release(nxt_unit_process_t *process); @@ -144,6 +148,8 @@ nxt_inline void nxt_unit_port_use(nxt_unit_port_t *port); nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port); static nxt_unit_port_t *nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue); +static void nxt_unit_process_awaiting_req(nxt_unit_ctx_t *ctx, + nxt_queue_t *awaiting_req); static void nxt_unit_remove_port(nxt_unit_impl_t *lib, nxt_unit_port_id_t *port_id); static nxt_unit_port_t *nxt_unit_remove_port_unsafe(nxt_unit_impl_t *lib, @@ -305,6 +311,9 @@ struct nxt_unit_ctx_impl_s { /* of nxt_unit_read_buf_t */ nxt_queue_t free_rbuf; + int online; + int ready; + nxt_unit_mmap_buf_t ctx_buf[2]; nxt_unit_read_buf_t ctx_read_buf; @@ -314,6 +323,7 @@ struct nxt_unit_ctx_impl_s { struct nxt_unit_mmap_s { nxt_port_mmap_header_t *hdr; + pthread_t src_thread; /* of nxt_unit_read_buf_t */ nxt_queue_t awaiting_rbuf; @@ -353,7 +363,6 @@ struct nxt_unit_impl_s { pid_t pid; int log_fd; - int online; nxt_unit_ctx_impl_t main_ctx; }; @@ -560,7 +569,6 @@ nxt_unit_create(nxt_unit_init_t *init) lib->ports.slot = NULL; lib->log_fd = STDERR_FILENO; - lib->online = 1; nxt_queue_init(&lib->contexts); @@ -614,10 +622,16 @@ nxt_unit_ctx_init(nxt_unit_impl_t *lib, nxt_unit_ctx_impl_t *ctx_impl, nxt_unit_lib_use(lib); + pthread_mutex_lock(&lib->mutex); + nxt_queue_insert_tail(&lib->contexts, &ctx_impl->link); + pthread_mutex_unlock(&lib->mutex); + ctx_impl->use_count = 1; ctx_impl->wait_items = 0; + ctx_impl->online = 1; + ctx_impl->ready = 0; nxt_queue_init(&ctx_impl->free_req); nxt_queue_init(&ctx_impl->free_ws); @@ -769,7 +783,7 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, uint32_t *shm_limit) { int rc; - int ready_fd, router_fd, read_fd; + int ready_fd, router_fd, read_in_fd, read_out_fd; char *unit_init, *version_end; long version_length; int64_t ready_pid, router_pid, read_pid; @@ -801,15 +815,15 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, "%"PRIu32";" "%"PRId64",%"PRIu32",%d;" "%"PRId64",%"PRIu32",%d;" - "%"PRId64",%"PRIu32",%d;" + "%"PRId64",%"PRIu32",%d,%d;" "%d,%"PRIu32, &ready_stream, &ready_pid, &ready_id, &ready_fd, &router_pid, &router_id, &router_fd, - &read_pid, &read_id, &read_fd, + &read_pid, &read_id, &read_in_fd, &read_out_fd, log_fd, shm_limit); - if (nxt_slow_path(rc != 12)) { + if (nxt_slow_path(rc != 13)) { nxt_unit_alert(NULL, "failed to scan variables: %d", rc); return NXT_UNIT_ERROR; @@ -829,8 +843,8 @@ nxt_unit_read_env(nxt_unit_port_t *ready_port, nxt_unit_port_t *router_port, nxt_unit_port_id_init(&read_port->id, (pid_t) read_pid, read_id); - read_port->in_fd = read_fd; - read_port->out_fd = -1; + read_port->in_fd = read_in_fd; + read_port->out_fd = read_out_fd; read_port->data = NULL; *stream = ready_stream; @@ -891,7 +905,8 @@ nxt_unit_ready(nxt_unit_ctx_t *ctx, int ready_fd, uint32_t stream, int queue_fd) static int -nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) +nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf, + nxt_unit_request_info_t **preq) { int rc; pid_t pid; @@ -979,6 +994,10 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) switch (port_msg->type) { + case _NXT_PORT_MSG_RPC_READY: + rc = NXT_UNIT_OK; + break; + case _NXT_PORT_MSG_QUIT: nxt_unit_debug(ctx, "#%"PRIu32": quit", port_msg->stream); @@ -990,6 +1009,10 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) rc = nxt_unit_process_new_port(ctx, &recv_msg); break; + case _NXT_PORT_MSG_PORT_ACK: + rc = nxt_unit_ctx_ready(ctx); + break; + case _NXT_PORT_MSG_CHANGE_FILE: nxt_unit_debug(ctx, "#%"PRIu32": change_file: fd %d", port_msg->stream, recv_msg.fd[0]); @@ -1019,7 +1042,7 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) break; case _NXT_PORT_MSG_REQ_HEADERS: - rc = nxt_unit_process_req_headers(ctx, &recv_msg); + rc = nxt_unit_process_req_headers(ctx, &recv_msg, preq); break; case _NXT_PORT_MSG_REQ_BODY: @@ -1055,7 +1078,7 @@ nxt_unit_process_msg(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) break; default: - nxt_unit_debug(ctx, "#%"PRIu32": ignore message type: %d", + nxt_unit_alert(ctx, "#%"PRIu32": ignore message type: %d", port_msg->stream, (int) port_msg->type); rc = NXT_UNIT_ERROR; @@ -1118,7 +1141,7 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); - if (new_port_msg->id == (nxt_port_id_t) -1) { + if (new_port_msg->id == NXT_UNIT_SHARED_PORT_ID) { nxt_unit_port_id_init(&new_port.id, lib->pid, new_port_msg->id); new_port.in_fd = recv_msg->fd[0]; @@ -1160,11 +1183,31 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) return NXT_UNIT_ERROR; } - if (new_port_msg->id == (nxt_port_id_t) -1) { + if (new_port_msg->id == NXT_UNIT_SHARED_PORT_ID) { lib->shared_port = port; - } else { - nxt_unit_port_release(port); + return nxt_unit_ctx_ready(ctx); + } + + nxt_unit_port_release(port); + + return NXT_UNIT_OK; +} + + +static int +nxt_unit_ctx_ready(nxt_unit_ctx_t *ctx) +{ + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + ctx_impl->ready = 1; + + if (lib->callbacks.ready_handler) { + return lib->callbacks.ready_handler(ctx); } return NXT_UNIT_OK; @@ -1172,7 +1215,8 @@ nxt_unit_process_new_port(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) static int -nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) +nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg, + nxt_unit_request_info_t **preq) { int res; nxt_unit_impl_t *lib; @@ -1288,7 +1332,12 @@ nxt_unit_process_req_headers(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) } } - lib->callbacks.request_handler(req); + if (preq == NULL) { + lib->callbacks.request_handler(req); + + } else { + *preq = req; + } } return NXT_UNIT_OK; @@ -1318,8 +1367,14 @@ nxt_unit_process_req_body(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) if (recv_msg->incoming_buf != NULL) { b = nxt_container_of(req->content_buf, nxt_unit_mmap_buf_t, buf); + while (b->next != NULL) { + b = b->next; + } + /* "Move" incoming buffer list to req_impl. */ - nxt_unit_mmap_buf_insert_tail(&b->next, recv_msg->incoming_buf); + b->next = recv_msg->incoming_buf; + b->next->prev = &b->next; + recv_msg->incoming_buf = NULL; } @@ -1588,8 +1643,6 @@ nxt_unit_process_websocket(nxt_unit_ctx_t *ctx, nxt_unit_recv_msg_t *recv_msg) } if (recv_msg->last) { - req_impl->websocket = 0; - if (cb->close_handler) { nxt_unit_req_debug(req, "close_handler"); @@ -1682,8 +1735,6 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) nxt_unit_request_hash_find(req->ctx, req_impl->stream, 1); } - req_impl->websocket = 0; - while (req_impl->outgoing_buf != NULL) { nxt_unit_mmap_buf_free(req_impl->outgoing_buf); } @@ -1704,6 +1755,8 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) req->response_port = NULL; } + req_impl->state = NXT_UNIT_RS_RELEASED; + pthread_mutex_lock(&ctx_impl->mutex); nxt_queue_remove(&req_impl->link); @@ -1711,8 +1764,6 @@ nxt_unit_request_info_release(nxt_unit_request_info_t *req) nxt_queue_insert_tail(&ctx_impl->free_req, &req_impl->link); pthread_mutex_unlock(&ctx_impl->mutex); - - req_impl->state = NXT_UNIT_RS_RELEASED; } @@ -2132,7 +2183,8 @@ nxt_unit_response_add_field(nxt_unit_request_info_t *req, resp = req->response; if (nxt_slow_path(resp->fields_count >= req->response_max_fields)) { - nxt_unit_req_warn(req, "add_field: too many response fields"); + nxt_unit_req_warn(req, "add_field: too many response fields (%d)", + (int) resp->fields_count); return NXT_UNIT_ERROR; } @@ -2309,6 +2361,8 @@ nxt_unit_response_buf_alloc(nxt_unit_request_info_t *req, uint32_t size) if (nxt_slow_path(rc != NXT_UNIT_OK)) { nxt_unit_mmap_buf_release(mmap_buf); + nxt_unit_req_alert(req, "response_buf_alloc: failed to get out buf"); + return NULL; } @@ -2949,8 +3003,6 @@ nxt_unit_request_read(nxt_unit_request_info_t *req, void *dst, size_t size) buf_res = nxt_unit_buf_read(&req->content_buf, &req->content_length, dst, size); - nxt_unit_req_debug(req, "read: %d", (int) buf_res); - if (buf_res < (ssize_t) size && req->content_fd != -1) { res = read(req->content_fd, dst, size); if (nxt_slow_path(res < 0)) { @@ -3389,7 +3441,10 @@ retry: for (mm = lib->outgoing.elts; mm < mm_end; mm++) { hdr = mm->hdr; - if (hdr->sent_over != 0xFFFFu && hdr->sent_over != port->id.id) { + if (hdr->sent_over != 0xFFFFu + && (hdr->sent_over != port->id.id + || mm->src_thread != pthread_self())) + { continue; } @@ -3657,6 +3712,7 @@ nxt_unit_new_mmap(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, int n) hdr->src_pid = lib->pid; hdr->dst_pid = port->id.pid; hdr->sent_over = port->id.id; + mm->src_thread = pthread_self(); /* Mark first n chunk(s) as busy */ for (i = 0; i < n; i++) { @@ -3975,6 +4031,8 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + nxt_unit_awake_ctx(ctx, ctx_impl); + } nxt_queue_loop; return rc; @@ -3982,6 +4040,32 @@ nxt_unit_incoming_mmap(nxt_unit_ctx_t *ctx, pid_t pid, int fd) static void +nxt_unit_awake_ctx(nxt_unit_ctx_t *ctx, nxt_unit_ctx_impl_t *ctx_impl) +{ + nxt_port_msg_t msg; + + if (nxt_fast_path(ctx == &ctx_impl->ctx)) { + return; + } + + if (nxt_slow_path(ctx_impl->read_port == NULL + || ctx_impl->read_port->out_fd == -1)) + { + nxt_unit_alert(ctx, "target context read_port is NULL or not writable"); + + return; + } + + memset(&msg, 0, sizeof(nxt_port_msg_t)); + + msg.type = _NXT_PORT_MSG_RPC_READY; + + (void) nxt_unit_port_send(ctx, ctx_impl->read_port, + &msg, sizeof(msg), NULL, 0); +} + + +static void nxt_unit_mmaps_init(nxt_unit_mmaps_t *mmaps) { pthread_mutex_init(&mmaps->mutex, NULL); @@ -4403,18 +4487,20 @@ nxt_unit_process_pop_first(nxt_unit_impl_t *lib) int nxt_unit_run(nxt_unit_ctx_t *ctx) { - int rc; - nxt_unit_impl_t *lib; + int rc; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + rc = NXT_UNIT_OK; - while (nxt_fast_path(lib->online)) { + while (nxt_fast_path(ctx_impl->online)) { rc = nxt_unit_run_once_impl(ctx); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + nxt_unit_quit(ctx); break; } } @@ -4458,7 +4544,7 @@ nxt_unit_run_once_impl(nxt_unit_ctx_t *ctx) return rc; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; } @@ -4483,17 +4569,17 @@ nxt_unit_read_buf(nxt_unit_ctx_t *ctx, nxt_unit_read_buf_t *rbuf) nxt_unit_port_impl_t *port_impl; struct pollfd fds[2]; - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); - if (ctx_impl->wait_items > 0 || lib->shared_port == NULL) { - + if (ctx_impl->wait_items > 0 || ctx_impl->ready == 0) { return nxt_unit_ctx_port_recv(ctx, ctx_impl->read_port, rbuf); } port_impl = nxt_container_of(ctx_impl->read_port, nxt_unit_port_impl_t, port); + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + retry: if (port_impl->from_socket == 0) { @@ -4585,8 +4671,6 @@ nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_read_buf_t *rbuf; - nxt_queue_init(&pending_rbuf); - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); pthread_mutex_lock(&ctx_impl->mutex); @@ -4597,6 +4681,8 @@ nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) return NXT_UNIT_OK; } + nxt_queue_init(&pending_rbuf); + nxt_queue_add(&pending_rbuf, &ctx_impl->pending_rbuf); nxt_queue_init(&ctx_impl->pending_rbuf); @@ -4607,7 +4693,7 @@ nxt_unit_process_pending_rbuf(nxt_unit_ctx_t *ctx) nxt_queue_each(rbuf, &pending_rbuf, nxt_unit_read_buf_t, link) { if (nxt_fast_path(rc != NXT_UNIT_ERROR)) { - rc = nxt_unit_process_msg(&ctx_impl->ctx, rbuf); + rc = nxt_unit_process_msg(&ctx_impl->ctx, rbuf, NULL); } else { nxt_unit_read_buf_release(ctx, rbuf); @@ -4629,8 +4715,6 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) nxt_unit_request_info_t *req; nxt_unit_request_info_impl_t *req_impl; - nxt_queue_init(&ready_req); - ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); pthread_mutex_lock(&ctx_impl->mutex); @@ -4641,6 +4725,8 @@ nxt_unit_process_ready_req(nxt_unit_ctx_t *ctx) return; } + nxt_queue_init(&ready_req); + nxt_queue_add(&ready_req, &ctx_impl->ready_req); nxt_queue_init(&ctx_impl->ready_req); @@ -4691,18 +4777,16 @@ int nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) { int rc; - nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); - lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); rc = NXT_UNIT_OK; - while (nxt_fast_path(lib->online)) { + while (nxt_fast_path(ctx_impl->online)) { rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -4716,7 +4800,7 @@ nxt_unit_run_ctx(nxt_unit_ctx_t *ctx) goto retry; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } @@ -4797,13 +4881,16 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; nxt_unit_ctx_use(ctx); lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + rc = NXT_UNIT_OK; - while (nxt_fast_path(lib->online)) { + while (nxt_fast_path(ctx_impl->online)) { rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { rc = NXT_UNIT_ERROR; @@ -4822,22 +4909,67 @@ nxt_unit_run_shared(nxt_unit_ctx_t *ctx) break; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { break; } + } - rc = nxt_unit_process_pending_rbuf(ctx); - if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { - break; - } + nxt_unit_ctx_release(ctx); - nxt_unit_process_ready_req(ctx); + return rc; +} + + +nxt_unit_request_info_t * +nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx) +{ + int rc; + nxt_unit_impl_t *lib; + nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_t *req; + + nxt_unit_ctx_use(ctx); + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + req = NULL; + + if (nxt_slow_path(!ctx_impl->online)) { + goto done; } + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + goto done; + } + + rc = nxt_unit_app_queue_recv(lib->shared_port, rbuf); + if (rc != NXT_UNIT_OK) { + nxt_unit_read_buf_release(ctx, rbuf); + goto done; + } + + (void) nxt_unit_process_msg(ctx, rbuf, &req); + +done: + nxt_unit_ctx_release(ctx); - return rc; + return req; +} + + +int +nxt_unit_is_main_ctx(nxt_unit_ctx_t *ctx) +{ + nxt_unit_impl_t *lib; + + lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + + return (ctx == &lib->main_ctx.ctx); } @@ -4862,6 +4994,7 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) int rc; nxt_unit_impl_t *lib; nxt_unit_read_buf_t *rbuf; + nxt_unit_ctx_impl_t *ctx_impl; rbuf = nxt_unit_read_buf_get(ctx); if (nxt_slow_path(rbuf == NULL)) { @@ -4869,6 +5002,7 @@ nxt_unit_process_port_msg_impl(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) } lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); retry: @@ -4884,7 +5018,7 @@ retry: return rc; } - rc = nxt_unit_process_msg(ctx, rbuf); + rc = nxt_unit_process_msg(ctx, rbuf, NULL); if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return NXT_UNIT_ERROR; } @@ -4896,12 +5030,12 @@ retry: nxt_unit_process_ready_req(ctx); - rbuf = nxt_unit_read_buf_get(ctx); - if (nxt_slow_path(rbuf == NULL)) { - return NXT_UNIT_ERROR; - } + if (ctx_impl->online) { + rbuf = nxt_unit_read_buf_get(ctx); + if (nxt_slow_path(rbuf == NULL)) { + return NXT_UNIT_ERROR; + } - if (lib->online) { goto retry; } @@ -4945,14 +5079,14 @@ nxt_unit_ctx_alloc(nxt_unit_ctx_t *ctx, void *data) queue_fd = -1; - port = nxt_unit_create_port(ctx); + port = nxt_unit_create_port(&new_ctx->ctx); if (nxt_slow_path(port == NULL)) { goto fail; } new_ctx->read_port = port; - queue_fd = nxt_unit_shm_open(ctx, sizeof(nxt_port_queue_t)); + queue_fd = nxt_unit_shm_open(&new_ctx->ctx, sizeof(nxt_port_queue_t)); if (nxt_slow_path(queue_fd == -1)) { goto fail; } @@ -4971,7 +5105,7 @@ nxt_unit_ctx_alloc(nxt_unit_ctx_t *ctx, void *data) port_impl = nxt_container_of(port, nxt_unit_port_impl_t, port); port_impl->queue = mem; - rc = nxt_unit_send_port(ctx, lib->router_port, port, queue_fd); + rc = nxt_unit_send_port(&new_ctx->ctx, lib->router_port, port, queue_fd); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } @@ -4997,6 +5131,7 @@ nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl) { nxt_unit_impl_t *lib; nxt_unit_mmap_buf_t *mmap_buf; + nxt_unit_read_buf_t *rbuf; nxt_unit_request_info_impl_t *req_impl; nxt_unit_websocket_frame_impl_t *ws_impl; @@ -5034,10 +5169,21 @@ nxt_unit_ctx_free(nxt_unit_ctx_impl_t *ctx_impl) } nxt_queue_loop; + nxt_queue_each(rbuf, &ctx_impl->free_rbuf, nxt_unit_read_buf_t, link) + { + if (rbuf != &ctx_impl->ctx_read_buf) { + nxt_unit_free(&ctx_impl->ctx, rbuf); + } + } nxt_queue_loop; + pthread_mutex_destroy(&ctx_impl->mutex); + pthread_mutex_lock(&lib->mutex); + nxt_queue_remove(&ctx_impl->link); + pthread_mutex_unlock(&lib->mutex); + if (nxt_fast_path(ctx_impl->read_port != NULL)) { nxt_unit_remove_port(lib, &ctx_impl->read_port->id); nxt_unit_port_release(ctx_impl->read_port); @@ -5224,7 +5370,7 @@ nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port) } if (port_impl->queue != NULL) { - munmap(port_impl->queue, (port->id.id == (nxt_port_id_t) -1) + munmap(port_impl->queue, (port->id.id == NXT_UNIT_SHARED_PORT_ID) ? sizeof(nxt_app_queue_t) : sizeof(nxt_port_queue_t)); } @@ -5237,14 +5383,12 @@ nxt_inline void nxt_unit_port_release(nxt_unit_port_t *port) static nxt_unit_port_t * nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) { - int rc; - nxt_queue_t awaiting_req; - nxt_unit_impl_t *lib; - nxt_unit_port_t *old_port; - nxt_unit_process_t *process; - nxt_unit_ctx_impl_t *ctx_impl; - nxt_unit_port_impl_t *new_port, *old_port_impl; - nxt_unit_request_info_impl_t *req_impl; + int rc, ready; + nxt_queue_t awaiting_req; + nxt_unit_impl_t *lib; + nxt_unit_port_t *old_port; + nxt_unit_process_t *process; + nxt_unit_port_impl_t *new_port, *old_port_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); @@ -5293,44 +5437,45 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) old_port_impl->queue = queue; } - if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { - nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); - nxt_queue_init(&old_port_impl->awaiting_req); - } - - old_port_impl->ready = (port->in_fd != -1 || port->out_fd != -1); + ready = (port->in_fd != -1 || port->out_fd != -1); - pthread_mutex_unlock(&lib->mutex); + /* + * Port can be market as 'ready' only after callbacks.add_port() call. + * Otherwise, request may try to use the port before callback. + */ + if (lib->callbacks.add_port == NULL && ready) { + old_port_impl->ready = ready; - if (lib->callbacks.add_port != NULL - && (port->in_fd != -1 || port->out_fd != -1)) - { - lib->callbacks.add_port(ctx, old_port); + if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { + nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); + nxt_queue_init(&old_port_impl->awaiting_req); + } } - nxt_queue_each(req_impl, &awaiting_req, - nxt_unit_request_info_impl_t, port_wait_link) - { - nxt_queue_remove(&req_impl->port_wait_link); + pthread_mutex_unlock(&lib->mutex); - ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t, - ctx); + if (lib->callbacks.add_port != NULL && ready) { + lib->callbacks.add_port(ctx, old_port); - pthread_mutex_lock(&ctx_impl->mutex); + pthread_mutex_lock(&lib->mutex); - nxt_queue_insert_tail(&ctx_impl->ready_req, - &req_impl->port_wait_link); + old_port_impl->ready = ready; - pthread_mutex_unlock(&ctx_impl->mutex); + if (!nxt_queue_is_empty(&old_port_impl->awaiting_req)) { + nxt_queue_add(&awaiting_req, &old_port_impl->awaiting_req); + nxt_queue_init(&old_port_impl->awaiting_req); + } - nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + pthread_mutex_unlock(&lib->mutex); + } - } nxt_queue_loop; + nxt_unit_process_awaiting_req(ctx, &awaiting_req); return old_port; } new_port = NULL; + ready = 0; nxt_unit_debug(ctx, "add_port: port{%d,%d} in_fd %d out_fd %d queue %p", port->id.pid, port->id.id, @@ -5341,7 +5486,9 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) goto unlock; } - if (port->id.id >= process->next_port_id) { + if (port->id.id != NXT_UNIT_SHARED_PORT_ID + && port->id.id >= process->next_port_id) + { process->next_port_id = port->id.id + 1; } @@ -5371,13 +5518,21 @@ nxt_unit_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port, void *queue) new_port->use_count = 2; new_port->process = process; - new_port->ready = (port->in_fd != -1 || port->out_fd != -1); new_port->queue = queue; new_port->from_socket = 0; new_port->socket_rbuf = NULL; nxt_queue_init(&new_port->awaiting_req); + ready = (port->in_fd != -1 || port->out_fd != -1); + + if (lib->callbacks.add_port == NULL) { + new_port->ready = ready; + + } else { + new_port->ready = 0; + } + process = NULL; unlock: @@ -5388,14 +5543,55 @@ unlock: nxt_unit_process_release(process); } - if (lib->callbacks.add_port != NULL - && new_port != NULL - && (port->in_fd != -1 || port->out_fd != -1)) - { + if (lib->callbacks.add_port != NULL && new_port != NULL && ready) { lib->callbacks.add_port(ctx, &new_port->port); + + nxt_queue_init(&awaiting_req); + + pthread_mutex_lock(&lib->mutex); + + new_port->ready = 1; + + if (!nxt_queue_is_empty(&new_port->awaiting_req)) { + nxt_queue_add(&awaiting_req, &new_port->awaiting_req); + nxt_queue_init(&new_port->awaiting_req); + } + + pthread_mutex_unlock(&lib->mutex); + + nxt_unit_process_awaiting_req(ctx, &awaiting_req); } - return &new_port->port; + return (new_port == NULL) ? NULL : &new_port->port; +} + + +static void +nxt_unit_process_awaiting_req(nxt_unit_ctx_t *ctx, nxt_queue_t *awaiting_req) +{ + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_request_info_impl_t *req_impl; + + nxt_queue_each(req_impl, awaiting_req, + nxt_unit_request_info_impl_t, port_wait_link) + { + nxt_queue_remove(&req_impl->port_wait_link); + + ctx_impl = nxt_container_of(req_impl->req.ctx, nxt_unit_ctx_impl_t, + ctx); + + pthread_mutex_lock(&ctx_impl->mutex); + + nxt_queue_insert_tail(&ctx_impl->ready_req, + &req_impl->port_wait_link); + + pthread_mutex_unlock(&ctx_impl->mutex); + + nxt_atomic_fetch_add(&ctx_impl->wait_items, -1); + + nxt_unit_awake_ctx(ctx, ctx_impl); + + } nxt_queue_loop; } @@ -5509,17 +5705,72 @@ nxt_unit_remove_process(nxt_unit_impl_t *lib, nxt_unit_process_t *process) static void nxt_unit_quit(nxt_unit_ctx_t *ctx) { - nxt_unit_impl_t *lib; + nxt_port_msg_t msg; + nxt_unit_impl_t *lib; + nxt_unit_ctx_impl_t *ctx_impl; + nxt_unit_callbacks_t *cb; + nxt_unit_request_info_t *req; + nxt_unit_request_info_impl_t *req_impl; lib = nxt_container_of(ctx->unit, nxt_unit_impl_t, unit); + ctx_impl = nxt_container_of(ctx, nxt_unit_ctx_impl_t, ctx); + + if (!ctx_impl->online) { + return; + } + + ctx_impl->online = 0; + + cb = &lib->callbacks; + + if (cb->quit != NULL) { + cb->quit(ctx); + } + + nxt_queue_each(req_impl, &ctx_impl->active_req, + nxt_unit_request_info_impl_t, link) + { + req = &req_impl->req; + + nxt_unit_req_warn(req, "active request on ctx quit"); - if (lib->online) { - lib->online = 0; + if (cb->close_handler) { + nxt_unit_req_debug(req, "close_handler"); + + cb->close_handler(req); - if (lib->callbacks.quit != NULL) { - lib->callbacks.quit(ctx); + } else { + nxt_unit_request_done(req, NXT_UNIT_ERROR); } + + } nxt_queue_loop; + + if (ctx != &lib->main_ctx.ctx) { + return; } + + memset(&msg, 0, sizeof(nxt_port_msg_t)); + + msg.pid = lib->pid; + msg.type = _NXT_PORT_MSG_QUIT; + + pthread_mutex_lock(&lib->mutex); + + nxt_queue_each(ctx_impl, &lib->contexts, nxt_unit_ctx_impl_t, link) { + + if (ctx == &ctx_impl->ctx + || ctx_impl->read_port == NULL + || ctx_impl->read_port->out_fd == -1) + { + continue; + } + + (void) nxt_unit_port_send(ctx, ctx_impl->read_port, + &msg, sizeof(msg), NULL, 0); + + } nxt_queue_loop; + + pthread_mutex_unlock(&lib->mutex); } @@ -6388,7 +6639,9 @@ nxt_unit_malloc(nxt_unit_ctx_t *ctx, size_t size) p = malloc(size); if (nxt_fast_path(p != NULL)) { +#if (NXT_DEBUG_ALLOC) nxt_unit_debug(ctx, "malloc(%d): %p", (int) size, p); +#endif } else { nxt_unit_alert(ctx, "malloc(%d) failed: %s (%d)", @@ -6402,7 +6655,9 @@ nxt_unit_malloc(nxt_unit_ctx_t *ctx, size_t size) void nxt_unit_free(nxt_unit_ctx_t *ctx, void *p) { +#if (NXT_DEBUG_ALLOC) nxt_unit_debug(ctx, "free(%p)", p); +#endif free(p); } diff --git a/src/nxt_unit.h b/src/nxt_unit.h index e90f0781..1e1a8dbe 100644 --- a/src/nxt_unit.h +++ b/src/nxt_unit.h @@ -12,6 +12,7 @@ #include <sys/uio.h> #include <string.h> +#include "nxt_auto_config.h" #include "nxt_version.h" #include "nxt_unit_typedefs.h" @@ -34,6 +35,8 @@ enum { #define NXT_UNIT_INIT_ENV "NXT_UNIT_INIT" +#define NXT_UNIT_SHARED_PORT_ID ((uint16_t) 0xFFFFu) + /* * Mostly opaque structure with library state. * @@ -152,6 +155,8 @@ struct nxt_unit_callbacks_s { /* Receive data on port id. Optional. */ ssize_t (*port_recv)(nxt_unit_ctx_t *, nxt_unit_port_t *port, void *buf, size_t buf_size, void *oob, size_t oob_size); + + int (*ready_handler)(nxt_unit_ctx_t *); }; @@ -208,6 +213,10 @@ int nxt_unit_run_ctx(nxt_unit_ctx_t *ctx); int nxt_unit_run_shared(nxt_unit_ctx_t *ctx); +nxt_unit_request_info_t *nxt_unit_dequeue_request(nxt_unit_ctx_t *ctx); + +int nxt_unit_is_main_ctx(nxt_unit_ctx_t *ctx); + /* * Receive and process one message, invoke configured callbacks. * diff --git a/src/perl/nxt_perl_psgi.c b/src/perl/nxt_perl_psgi.c index 16079a38..5df1465d 100644 --- a/src/perl/nxt_perl_psgi.c +++ b/src/perl/nxt_perl_psgi.c @@ -18,14 +18,14 @@ typedef struct { PerlInterpreter *my_perl; - nxt_unit_request_info_t *req; -} nxt_perl_psgi_input_t; - - -typedef struct { - PerlInterpreter *my_perl; + nxt_perl_psgi_io_arg_t arg_input; + nxt_perl_psgi_io_arg_t arg_error; SV *app; -} nxt_perl_psgi_module_t; + CV *cb; + nxt_unit_request_info_t *req; + pthread_t thread; + nxt_unit_ctx_t *ctx; +} nxt_perl_psgi_ctx_t; static long nxt_perl_psgi_io_input_read(PerlInterpreter *my_perl, @@ -62,11 +62,11 @@ static nxt_int_t nxt_perl_psgi_io_input_init(PerlInterpreter *my_perl, static nxt_int_t nxt_perl_psgi_io_error_init(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg); -static PerlInterpreter *nxt_perl_psgi_interpreter_init(nxt_task_t *task, - char *script, SV **app); +static int nxt_perl_psgi_ctx_init(const char *script, + nxt_perl_psgi_ctx_t *pctx); static SV *nxt_perl_psgi_env_create(PerlInterpreter *my_perl, - nxt_unit_request_info_t *req, nxt_perl_psgi_input_t *input); + nxt_unit_request_info_t *req); nxt_inline int nxt_perl_psgi_add_sptr(PerlInterpreter *my_perl, HV *hash_env, const char *name, uint32_t name_len, nxt_unit_sptr_t *sptr, uint32_t len); nxt_inline int nxt_perl_psgi_add_str(PerlInterpreter *my_perl, HV *hash_env, @@ -75,8 +75,7 @@ nxt_inline int nxt_perl_psgi_add_value(PerlInterpreter *my_perl, HV *hash_env, const char *name, uint32_t name_len, void *value); -static u_char *nxt_perl_psgi_module_create(nxt_task_t *task, - const char *script); +static char *nxt_perl_psgi_module_create(const char *script); static nxt_int_t nxt_perl_psgi_result_status(PerlInterpreter *my_perl, SV *result); @@ -96,18 +95,20 @@ static void nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, nxt_unit_request_info_t *req); static nxt_int_t nxt_perl_psgi_start(nxt_task_t *task, - nxt_process_data_t *conf); + nxt_process_data_t *data); static void nxt_perl_psgi_request_handler(nxt_unit_request_info_t *req); -static void nxt_perl_psgi_atexit(void); - -typedef SV *(*nxt_perl_psgi_callback_f)(PerlInterpreter *my_perl, - SV *env, nxt_task_t *task); - -static CV *nxt_perl_psgi_cb; -static PerlInterpreter *nxt_perl_psgi; -static nxt_perl_psgi_io_arg_t nxt_perl_psgi_arg_input; -static nxt_perl_psgi_io_arg_t nxt_perl_psgi_arg_error; -static nxt_unit_request_info_t *nxt_perl_psgi_request; +static int nxt_perl_psgi_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_perl_psgi_thread_func(void *main_ctx); +static int nxt_perl_psgi_init_threads(nxt_perl_app_conf_t *c); +static void nxt_perl_psgi_join_threads(nxt_unit_ctx_t *ctx, + nxt_perl_app_conf_t *c); +static void nxt_perl_psgi_ctx_free(nxt_perl_psgi_ctx_t *pctx); + +static CV *nxt_perl_psgi_write; +static CV *nxt_perl_psgi_close; +static CV *nxt_perl_psgi_cb; +static pthread_attr_t *nxt_perl_psgi_thread_attr; +static nxt_perl_psgi_ctx_t *nxt_perl_psgi_ctxs; static uint32_t nxt_perl_psgi_compat[] = { NXT_VERNUM, NXT_DEBUG, @@ -129,11 +130,11 @@ static long nxt_perl_psgi_io_input_read(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg, void *vbuf, size_t length) { - nxt_perl_psgi_input_t *input; + nxt_perl_psgi_ctx_t *pctx; - input = (nxt_perl_psgi_input_t *) arg->ctx; + pctx = arg->pctx; - return nxt_unit_request_read(input->req, vbuf, length); + return nxt_unit_request_read(pctx->req, vbuf, length); } @@ -165,10 +166,11 @@ static long nxt_perl_psgi_io_error_write(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg, const void *vbuf, size_t length) { - nxt_perl_psgi_input_t *input; + nxt_perl_psgi_ctx_t *pctx; - input = (nxt_perl_psgi_input_t *) arg->ctx; - nxt_unit_req_error(input->req, "Perl: %s", (const char*) vbuf); + pctx = arg->pctx; + + nxt_unit_req_error(pctx->req, "Perl: %s", (const char*) vbuf); return (long) length; } @@ -216,9 +218,10 @@ XS(XS_NGINX__Unit__PSGI_exit) XS(XS_NGINX__Unit__Sandbox_write); XS(XS_NGINX__Unit__Sandbox_write) { - int rc; - char *body; - size_t len; + int rc; + char *body; + size_t len; + nxt_perl_psgi_ctx_t *pctx; dXSARGS; @@ -230,7 +233,9 @@ XS(XS_NGINX__Unit__Sandbox_write) body = SvPV(ST(1), len); - rc = nxt_unit_response_write(nxt_perl_psgi_request, body, len); + pctx = CvXSUBANY(cv).any_ptr; + + rc = nxt_unit_response_write(pctx->req, body, len); if (nxt_slow_path(rc != NXT_UNIT_OK)) { Perl_croak(aTHX_ "Failed to write response body"); @@ -242,15 +247,11 @@ XS(XS_NGINX__Unit__Sandbox_write) nxt_inline void -nxt_perl_psgi_cb_request_done(nxt_int_t status) +nxt_perl_psgi_cb_request_done(nxt_perl_psgi_ctx_t *pctx, int status) { - nxt_unit_request_info_t *req; - - req = nxt_perl_psgi_request; - - if (req != NULL) { - nxt_unit_request_done(req, status); - nxt_perl_psgi_request = NULL; + if (pctx->req != NULL) { + nxt_unit_request_done(pctx->req, status); + pctx->req = NULL; } } @@ -262,7 +263,7 @@ XS(XS_NGINX__Unit__Sandbox_close) ax = POPMARK; - nxt_perl_psgi_cb_request_done(NXT_UNIT_OK); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_OK); XSRETURN_NO; } @@ -271,14 +272,15 @@ XS(XS_NGINX__Unit__Sandbox_close) XS(XS_NGINX__Unit__Sandbox_cb); XS(XS_NGINX__Unit__Sandbox_cb) { - SV *obj; - int rc; - long array_len; + SV *obj; + int rc; + long array_len; + nxt_perl_psgi_ctx_t *pctx; dXSARGS; if (nxt_slow_path(items != 1)) { - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); Perl_croak(aTHX_ "Wrong number of arguments"); @@ -288,7 +290,7 @@ XS(XS_NGINX__Unit__Sandbox_cb) if (nxt_slow_path(SvOK(ST(0)) == 0 || SvROK(ST(0)) == 0 || SvTYPE(SvRV(ST(0))) != SVt_PVAV)) { - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); Perl_croak(aTHX_ "PSGI: An unexpected response was received " "from Perl Application"); @@ -296,10 +298,11 @@ XS(XS_NGINX__Unit__Sandbox_cb) XSRETURN_EMPTY; } - rc = nxt_perl_psgi_result_array(PERL_GET_CONTEXT, ST(0), - nxt_perl_psgi_request); + pctx = CvXSUBANY(cv).any_ptr; + + rc = nxt_perl_psgi_result_array(PERL_GET_CONTEXT, ST(0), pctx->req); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_ERROR); Perl_croak(aTHX_ (char *) NULL); @@ -316,7 +319,7 @@ XS(XS_NGINX__Unit__Sandbox_cb) XSRETURN(1); } - nxt_perl_psgi_cb_request_done(NXT_UNIT_OK); + nxt_perl_psgi_cb_request_done(CvXSUBANY(cv).any_ptr, NXT_UNIT_OK); XSRETURN_EMPTY; } @@ -335,10 +338,11 @@ nxt_perl_psgi_xs_init(pTHX) /* DynaLoader for Perl modules who use XS */ newXS("DynaLoader::boot_DynaLoader", boot_DynaLoader, __FILE__); - newXS("NGINX::Unit::Sandbox::write", XS_NGINX__Unit__Sandbox_write, - __FILE__); - newXS("NGINX::Unit::Sandbox::close", XS_NGINX__Unit__Sandbox_close, - __FILE__); + nxt_perl_psgi_write = newXS("NGINX::Unit::Sandbox::write", + XS_NGINX__Unit__Sandbox_write, __FILE__); + + nxt_perl_psgi_close = newXS("NGINX::Unit::Sandbox::close", + XS_NGINX__Unit__Sandbox_close, __FILE__); nxt_perl_psgi_cb = newXS("NGINX::Unit::Sandbox::cb", XS_NGINX__Unit__Sandbox_cb, __FILE__); @@ -416,10 +420,10 @@ nxt_perl_psgi_call_method(PerlInterpreter *my_perl, SV *obj, const char *method, } -static u_char * -nxt_perl_psgi_module_create(nxt_task_t *task, const char *script) +static char * +nxt_perl_psgi_module_create(const char *script) { - u_char *buf, *p; + char *buf, *p; size_t length; static nxt_str_t prefix = nxt_string( @@ -441,12 +445,11 @@ nxt_perl_psgi_module_create(nxt_task_t *task, const char *script) length = strlen(script); - buf = nxt_malloc(prefix.length + length + suffix.length); - + buf = nxt_unit_malloc(NULL, prefix.length + length + suffix.length); if (nxt_slow_path(buf == NULL)) { - nxt_log_error(NXT_LOG_ERR, task->log, - "PSGI: Failed to allocate memory " - "for Perl script file %s", script); + nxt_unit_alert(NULL, "PSGI: Failed to allocate memory " + "for Perl script file %s", script); + return NULL; } @@ -518,30 +521,27 @@ nxt_perl_psgi_io_error_init(PerlInterpreter *my_perl, } -static PerlInterpreter * -nxt_perl_psgi_interpreter_init(nxt_task_t *task, char *script, SV **app) +static int +nxt_perl_psgi_ctx_init(const char *script, nxt_perl_psgi_ctx_t *pctx) { - int status, pargc; - char **pargv, **penv; - u_char *run_module; + int status; + char *run_module; PerlInterpreter *my_perl; static char argv[] = "\0""-e\0""0"; static char *embedding[] = { &argv[0], &argv[1], &argv[4] }; - pargc = 0; - pargv = NULL; - penv = NULL; - - PERL_SYS_INIT3(&pargc, &pargv, &penv); - my_perl = perl_alloc(); if (nxt_slow_path(my_perl == NULL)) { - nxt_alert(task, "PSGI: Failed to allocate memory for Perl interpreter"); - return NULL; + nxt_unit_alert(NULL, + "PSGI: Failed to allocate memory for Perl interpreter"); + + return NXT_UNIT_ERROR; } + pctx->my_perl = my_perl; + run_module = NULL; perl_construct(my_perl); @@ -550,77 +550,86 @@ nxt_perl_psgi_interpreter_init(nxt_task_t *task, char *script, SV **app) status = perl_parse(my_perl, nxt_perl_psgi_xs_init, 3, embedding, NULL); if (nxt_slow_path(status != 0)) { - nxt_alert(task, "PSGI: Failed to parse Perl Script"); + nxt_unit_alert(NULL, "PSGI: Failed to parse Perl Script"); goto fail; } + CvXSUBANY(nxt_perl_psgi_write).any_ptr = pctx; + CvXSUBANY(nxt_perl_psgi_close).any_ptr = pctx; + CvXSUBANY(nxt_perl_psgi_cb).any_ptr = pctx; + + pctx->cb = nxt_perl_psgi_cb; + PL_exit_flags |= PERL_EXIT_DESTRUCT_END; PL_origalen = 1; status = perl_run(my_perl); if (nxt_slow_path(status != 0)) { - nxt_alert(task, "PSGI: Failed to run Perl"); + nxt_unit_alert(NULL, "PSGI: Failed to run Perl"); goto fail; } sv_setsv(get_sv("0", 0), newSVpv(script, 0)); - run_module = nxt_perl_psgi_module_create(task, script); - + run_module = nxt_perl_psgi_module_create(script); if (nxt_slow_path(run_module == NULL)) { goto fail; } - status = nxt_perl_psgi_io_input_init(my_perl, &nxt_perl_psgi_arg_input); + pctx->arg_input.pctx = pctx; + status = nxt_perl_psgi_io_input_init(my_perl, &pctx->arg_input); if (nxt_slow_path(status != NXT_OK)) { - nxt_alert(task, "PSGI: Failed to init io.psgi.input"); + nxt_unit_alert(NULL, "PSGI: Failed to init io.psgi.input"); goto fail; } - status = nxt_perl_psgi_io_error_init(my_perl, &nxt_perl_psgi_arg_error); + pctx->arg_error.pctx = pctx; + status = nxt_perl_psgi_io_error_init(my_perl, &pctx->arg_error); if (nxt_slow_path(status != NXT_OK)) { - nxt_alert(task, "PSGI: Failed to init io.psgi.errors"); + nxt_unit_alert(NULL, "PSGI: Failed to init io.psgi.errors"); goto fail; } - *app = eval_pv((const char *) run_module, FALSE); + pctx->app = eval_pv(run_module, FALSE); if (SvTRUE(ERRSV)) { - nxt_alert(task, "PSGI: Failed to parse script: %s\n%s", - script, SvPV_nolen(ERRSV)); + nxt_unit_alert(NULL, "PSGI: Failed to parse script: %s\n%s", + script, SvPV_nolen(ERRSV)); goto fail; } - nxt_free(run_module); + nxt_unit_free(NULL, run_module); - return my_perl; + return NXT_UNIT_OK; fail: if (run_module != NULL) { - nxt_free(run_module); + nxt_unit_free(NULL, run_module); } perl_destruct(my_perl); perl_free(my_perl); - PERL_SYS_TERM(); - return NULL; + return NXT_UNIT_ERROR; } static SV * nxt_perl_psgi_env_create(PerlInterpreter *my_perl, - nxt_unit_request_info_t *req, nxt_perl_psgi_input_t *input) + nxt_unit_request_info_t *req) { - HV *hash_env; - AV *array_version; - uint32_t i; - nxt_unit_field_t *f; - nxt_unit_request_t *r; + HV *hash_env; + AV *array_version; + uint32_t i; + nxt_unit_field_t *f; + nxt_unit_request_t *r; + nxt_perl_psgi_ctx_t *pctx; + + pctx = req->ctx->data; hash_env = newHV(); if (nxt_slow_path(hash_env == NULL)) { @@ -664,11 +673,12 @@ nxt_perl_psgi_env_create(PerlInterpreter *my_perl, : newSVpv("http", 4))); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.input"), - SvREFCNT_inc(nxt_perl_psgi_arg_input.io))); + SvREFCNT_inc(pctx->arg_input.io))); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.errors"), - SvREFCNT_inc(nxt_perl_psgi_arg_error.io))); + SvREFCNT_inc(pctx->arg_error.io))); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.multithread"), - &PL_sv_no)); + nxt_perl_psgi_ctxs != NULL + ? &PL_sv_yes : &PL_sv_no)); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.multiprocess"), &PL_sv_yes)); RC(nxt_perl_psgi_add_value(my_perl, hash_env, NL("psgi.run_once"), @@ -1109,13 +1119,17 @@ static void nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, nxt_unit_request_info_t *req) { + nxt_perl_psgi_ctx_t *pctx; + dSP; + pctx = req->ctx->data; + ENTER; SAVETMPS; PUSHMARK(sp); - XPUSHs(newRV_noinc((SV*) nxt_perl_psgi_cb)); + XPUSHs(newRV_noinc((SV*) pctx->cb)); PUTBACK; call_sv(result, G_EVAL|G_SCALAR); @@ -1126,7 +1140,7 @@ nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, nxt_unit_error(NULL, "PSGI: Failed to execute result callback: \n%s", SvPV_nolen(ERRSV)); - nxt_perl_psgi_cb_request_done(NXT_UNIT_ERROR); + nxt_perl_psgi_cb_request_done(pctx, NXT_UNIT_ERROR); } PUTBACK; @@ -1138,90 +1152,116 @@ nxt_perl_psgi_result_cb(PerlInterpreter *my_perl, SV *result, static nxt_int_t nxt_perl_psgi_start(nxt_task_t *task, nxt_process_data_t *data) { - int rc; - nxt_unit_ctx_t *unit_ctx; - nxt_unit_init_t perl_init; - PerlInterpreter *my_perl; - nxt_common_app_conf_t *conf; - nxt_perl_psgi_module_t module; + int rc, pargc; + char **pargv, **penv; + nxt_unit_ctx_t *unit_ctx; + nxt_unit_init_t perl_init; + nxt_perl_psgi_ctx_t pctx; + nxt_perl_app_conf_t *c; + nxt_common_app_conf_t *common_conf; + + common_conf = data->app; + c = &common_conf->u.perl; + + pargc = 0; + pargv = NULL; + penv = NULL; - conf = data->app; + PERL_SYS_INIT3(&pargc, &pargv, &penv); - my_perl = nxt_perl_psgi_interpreter_init(task, conf->u.perl.script, - &module.app); + memset(&pctx, 0, sizeof(nxt_perl_psgi_ctx_t)); - if (nxt_slow_path(my_perl == NULL)) { - return NXT_ERROR; + rc = nxt_perl_psgi_ctx_init(c->script, &pctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; } - module.my_perl = my_perl; - nxt_perl_psgi = my_perl; + rc = nxt_perl_psgi_init_threads(c); + + PERL_SET_CONTEXT(pctx.my_perl); + + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; + } nxt_unit_default_init(task, &perl_init); perl_init.callbacks.request_handler = nxt_perl_psgi_request_handler; - perl_init.data = &module; - perl_init.shm_limit = conf->shm_limit; + perl_init.callbacks.ready_handler = nxt_perl_psgi_ready_handler; + perl_init.data = c; + perl_init.ctx_data = &pctx; + perl_init.shm_limit = common_conf->shm_limit; unit_ctx = nxt_unit_init(&perl_init); if (nxt_slow_path(unit_ctx == NULL)) { - return NXT_ERROR; + goto fail; } rc = nxt_unit_run(unit_ctx); + nxt_perl_psgi_join_threads(unit_ctx, c); + nxt_unit_done(unit_ctx); - nxt_perl_psgi_atexit(); + nxt_perl_psgi_ctx_free(&pctx); + + PERL_SYS_TERM(); exit(rc); return NXT_OK; + +fail: + + nxt_perl_psgi_join_threads(NULL, c); + + nxt_perl_psgi_ctx_free(&pctx); + + PERL_SYS_TERM(); + + return NXT_ERROR; } static void nxt_perl_psgi_request_handler(nxt_unit_request_info_t *req) { - SV *env, *result; - nxt_int_t rc; - PerlInterpreter *my_perl; - nxt_perl_psgi_input_t input; - nxt_perl_psgi_module_t *module; - - module = req->unit->data; - my_perl = module->my_perl; + SV *env, *result; + nxt_int_t rc; + PerlInterpreter *my_perl; + nxt_perl_psgi_ctx_t *pctx; - input.my_perl = my_perl; - input.req = req; + pctx = req->ctx->data; + my_perl = pctx->my_perl; - nxt_perl_psgi_request = req; + pctx->req = req; /* * Create environ variable for perl sub "application". * > sub application { * > my ($environ) = @_; */ - env = nxt_perl_psgi_env_create(my_perl, req, &input); + env = nxt_perl_psgi_env_create(my_perl, req); if (nxt_slow_path(env == NULL)) { nxt_unit_req_error(req, "PSGI: Failed to create 'env' for Perl Application"); nxt_unit_request_done(req, NXT_UNIT_ERROR); + pctx->req = NULL; return; } - nxt_perl_psgi_arg_input.ctx = &input; - nxt_perl_psgi_arg_error.ctx = &input; - /* Call perl sub and get result as SV*. */ - result = nxt_perl_psgi_call_var_application(my_perl, env, module->app, req); + result = nxt_perl_psgi_call_var_application(my_perl, env, pctx->app, + req); if (nxt_fast_path(SvOK(result) != 0 && SvROK(result) != 0)) { if (SvTYPE(SvRV(result)) == SVt_PVAV) { rc = nxt_perl_psgi_result_array(my_perl, result, req); nxt_unit_request_done(req, rc); + pctx->req = NULL; + goto release; } @@ -1235,6 +1275,7 @@ nxt_perl_psgi_request_handler(nxt_unit_request_info_t *req) "from Perl Application"); nxt_unit_request_done(req, NXT_UNIT_ERROR); + pctx->req = NULL; release: @@ -1243,18 +1284,181 @@ release: } +static int +nxt_perl_psgi_ready_handler(nxt_unit_ctx_t *ctx) +{ + int res; + uint32_t i; + nxt_perl_app_conf_t *c; + nxt_perl_psgi_ctx_t *pctx; + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + c = ctx->unit->data; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + pctx = &nxt_perl_psgi_ctxs[i]; + + pctx->ctx = ctx; + + res = pthread_create(&pctx->thread, nxt_perl_psgi_thread_attr, + nxt_perl_psgi_thread_func, pctx); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d create failed: %s (%d)", + (int) (i + 1), strerror(res), res); + + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void * +nxt_perl_psgi_thread_func(void *data) +{ + nxt_unit_ctx_t *ctx; + nxt_perl_psgi_ctx_t *pctx; + + pctx = data; + + nxt_unit_debug(pctx->ctx, "worker thread start"); + + ctx = nxt_unit_ctx_alloc(pctx->ctx, pctx); + if (nxt_slow_path(ctx == NULL)) { + return NULL; + } + + pctx->ctx = ctx; + + PERL_SET_CONTEXT(pctx->my_perl); + + (void) nxt_unit_run(ctx); + + nxt_unit_done(ctx); + + nxt_unit_debug(NULL, "worker thread end"); + + return NULL; +} + + +static int +nxt_perl_psgi_init_threads(nxt_perl_app_conf_t *c) +{ + int rc; + uint32_t i; + static pthread_attr_t attr; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + if (c->thread_stack_size > 0) { + rc = pthread_attr_init(&attr); + if (nxt_slow_path(rc != 0)) { + nxt_unit_alert(NULL, "thread attr init failed: %s (%d)", + strerror(rc), rc); + + return NXT_UNIT_ERROR; + } + + rc = pthread_attr_setstacksize(&attr, c->thread_stack_size); + if (nxt_slow_path(rc != 0)) { + nxt_unit_alert(NULL, "thread attr set stack size failed: %s (%d)", + strerror(rc), rc); + + return NXT_UNIT_ERROR; + } + + nxt_perl_psgi_thread_attr = &attr; + } + + nxt_perl_psgi_ctxs = nxt_unit_malloc(NULL, sizeof(nxt_perl_psgi_ctx_t) + * (c->threads - 1)); + if (nxt_slow_path(nxt_perl_psgi_ctxs == NULL)) { + return NXT_UNIT_ERROR; + } + + memset(nxt_perl_psgi_ctxs, 0, sizeof(nxt_perl_psgi_ctx_t) + * (c->threads - 1)); + + for (i = 0; i < c->threads - 1; i++) { + rc = nxt_perl_psgi_ctx_init(c->script, &nxt_perl_psgi_ctxs[i]); + + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + static void -nxt_perl_psgi_atexit(void) +nxt_perl_psgi_join_threads(nxt_unit_ctx_t *ctx, nxt_perl_app_conf_t *c) { - dTHXa(nxt_perl_psgi); + int res; + uint32_t i; + nxt_perl_psgi_ctx_t *pctx; - nxt_perl_psgi_layer_stream_io_destroy(aTHX_ nxt_perl_psgi_arg_input.io); - nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ nxt_perl_psgi_arg_input.fp); + if (nxt_perl_psgi_ctxs == NULL) { + return; + } - nxt_perl_psgi_layer_stream_io_destroy(aTHX_ nxt_perl_psgi_arg_error.io); - nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ nxt_perl_psgi_arg_error.fp); + for (i = 0; i < c->threads - 1; i++) { + pctx = &nxt_perl_psgi_ctxs[i]; - perl_destruct(nxt_perl_psgi); - perl_free(nxt_perl_psgi); - PERL_SYS_TERM(); + res = pthread_join(pctx->thread, NULL); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d joined", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d join failed: %s (%d)", + (int) (i + 1), strerror(res), res); + } + } + + for (i = 0; i < c->threads - 1; i++) { + nxt_perl_psgi_ctx_free(&nxt_perl_psgi_ctxs[i]); + } + + nxt_unit_free(NULL, nxt_perl_psgi_ctxs); +} + + +static void +nxt_perl_psgi_ctx_free(nxt_perl_psgi_ctx_t *pctx) +{ + PerlInterpreter *my_perl; + + my_perl = pctx->my_perl; + + if (nxt_slow_path(my_perl == NULL)) { + return; + } + + PERL_SET_CONTEXT(my_perl); + + nxt_perl_psgi_layer_stream_io_destroy(aTHX_ pctx->arg_input.io); + nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ pctx->arg_input.fp); + + nxt_perl_psgi_layer_stream_io_destroy(aTHX_ pctx->arg_error.io); + nxt_perl_psgi_layer_stream_fp_destroy(aTHX_ pctx->arg_error.fp); + + perl_destruct(my_perl); + perl_free(my_perl); } diff --git a/src/perl/nxt_perl_psgi_layer.h b/src/perl/nxt_perl_psgi_layer.h index 3fa349c0..af18ad0d 100644 --- a/src/perl/nxt_perl_psgi_layer.h +++ b/src/perl/nxt_perl_psgi_layer.h @@ -14,7 +14,7 @@ #include <perliol.h> -typedef struct nxt_perl_psgi_io_arg nxt_perl_psgi_io_arg_t; +typedef struct nxt_perl_psgi_io_arg_s nxt_perl_psgi_io_arg_t; typedef long (*nxt_perl_psgi_io_read_f)(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg, void *vbuf, size_t length); @@ -24,7 +24,7 @@ typedef long (*nxt_perl_psgi_io_arg_f)(PerlInterpreter *my_perl, nxt_perl_psgi_io_arg_t *arg); -struct nxt_perl_psgi_io_arg { +struct nxt_perl_psgi_io_arg_s { SV *io; PerlIO *fp; @@ -32,7 +32,7 @@ struct nxt_perl_psgi_io_arg { nxt_perl_psgi_io_read_f read; nxt_perl_psgi_io_write_f write; - void *ctx; + void *pctx; }; diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c index 01534a47..faf0c0e1 100644 --- a/src/python/nxt_python.c +++ b/src/python/nxt_python.c @@ -15,8 +15,20 @@ #include NXT_PYTHON_MOUNTS_H +typedef struct { + pthread_t thread; + nxt_unit_ctx_t *ctx; + void *ctx_data; +} nxt_py_thread_info_t; + + static nxt_int_t nxt_python_start(nxt_task_t *task, nxt_process_data_t *data); +static int nxt_python_init_threads(nxt_python_app_conf_t *c); +static int nxt_python_ready_handler(nxt_unit_ctx_t *ctx); +static void *nxt_python_thread_func(void *main_ctx); +static void nxt_python_join_threads(nxt_unit_ctx_t *ctx, + nxt_python_app_conf_t *c); static void nxt_python_atexit(void); static uint32_t compat[] = { @@ -44,14 +56,19 @@ static wchar_t *nxt_py_home; static char *nxt_py_home; #endif +static pthread_attr_t *nxt_py_thread_attr; +static nxt_py_thread_info_t *nxt_py_threads; +static nxt_python_proto_t nxt_py_proto; + static nxt_int_t nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) { - int rc, asgi; + int rc; char *nxt_py_module; size_t len; PyObject *obj, *pypath, *module; + nxt_str_t proto; const char *callable; nxt_unit_ctx_t *unit_ctx; nxt_unit_init_t python_init; @@ -66,6 +83,9 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) static const char bin_python[] = "/bin/python"; #endif + static const nxt_str_t wsgi = nxt_string("wsgi"); + static const nxt_str_t asgi = nxt_string("asgi"); + app_conf = data->app; c = &app_conf->u.python; @@ -124,9 +144,17 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) Py_InitializeEx(0); +#if PY_VERSION_HEX < NXT_PYTHON_VER(3, 7) + if (c->threads > 1) { + PyEval_InitThreads(); + } +#endif + module = NULL; obj = NULL; + python_init.ctx_data = NULL; + obj = PySys_GetObject((char *) "stderr"); if (nxt_slow_path(obj == NULL)) { nxt_alert(task, "Python failed to get \"sys.stderr\" object"); @@ -216,35 +244,56 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) nxt_unit_default_init(task, &python_init); + python_init.data = c; python_init.shm_limit = data->app->shm_limit; + python_init.callbacks.ready_handler = nxt_python_ready_handler; - asgi = nxt_python_asgi_check(nxt_py_application); + proto = c->protocol; - if (asgi) { - rc = nxt_python_asgi_init(task, &python_init); + if (proto.length == 0) { + proto = nxt_python_asgi_check(nxt_py_application) ? asgi : wsgi; + } + + if (nxt_strstr_eq(&proto, &asgi)) { + rc = nxt_python_asgi_init(&python_init, &nxt_py_proto); } else { - rc = nxt_python_wsgi_init(task, &python_init); + rc = nxt_python_wsgi_init(&python_init, &nxt_py_proto); + } + + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + goto fail; + } + + rc = nxt_py_proto.ctx_data_alloc(&python_init.ctx_data); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + goto fail; } - if (nxt_slow_path(rc == NXT_ERROR)) { + rc = nxt_python_init_threads(c); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { goto fail; } + if (nxt_py_proto.startup != NULL) { + if (nxt_py_proto.startup(python_init.ctx_data) != NXT_UNIT_OK) { + goto fail; + } + } + unit_ctx = nxt_unit_init(&python_init); if (nxt_slow_path(unit_ctx == NULL)) { goto fail; } - if (asgi) { - rc = nxt_python_asgi_run(unit_ctx); + rc = nxt_py_proto.run(unit_ctx); - } else { - rc = nxt_python_wsgi_run(unit_ctx); - } + nxt_python_join_threads(unit_ctx, c); nxt_unit_done(unit_ctx); + nxt_py_proto.ctx_data_free(python_init.ctx_data); + nxt_python_atexit(); exit(rc); @@ -253,6 +302,12 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) fail: + nxt_python_join_threads(NULL, c); + + if (python_init.ctx_data != NULL) { + nxt_py_proto.ctx_data_free(python_init.ctx_data); + } + Py_XDECREF(obj); Py_XDECREF(module); @@ -262,7 +317,195 @@ fail: } -nxt_int_t +static int +nxt_python_init_threads(nxt_python_app_conf_t *c) +{ + int res; + uint32_t i; + nxt_py_thread_info_t *ti; + static pthread_attr_t attr; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + if (c->thread_stack_size > 0) { + res = pthread_attr_init(&attr); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr init failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + res = pthread_attr_setstacksize(&attr, c->thread_stack_size); + if (nxt_slow_path(res != 0)) { + nxt_unit_alert(NULL, "thread attr set stack size failed: %s (%d)", + strerror(res), res); + + return NXT_UNIT_ERROR; + } + + nxt_py_thread_attr = &attr; + } + + nxt_py_threads = nxt_unit_malloc(NULL, sizeof(nxt_py_thread_info_t) + * (c->threads - 1)); + if (nxt_slow_path(nxt_py_threads == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate thread info array"); + + return NXT_UNIT_ERROR; + } + + memset(nxt_py_threads, 0, sizeof(nxt_py_thread_info_t) * (c->threads - 1)); + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + res = nxt_py_proto.ctx_data_alloc(&ti->ctx_data); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static int +nxt_python_ready_handler(nxt_unit_ctx_t *ctx) +{ + int res; + uint32_t i; + nxt_py_thread_info_t *ti; + nxt_python_app_conf_t *c; + + if (nxt_py_proto.ready != NULL) { + res = nxt_py_proto.ready(ctx); + if (nxt_slow_path(res != NXT_UNIT_OK)) { + return NXT_UNIT_ERROR; + } + } + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + c = ctx->unit->data; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + ti->ctx = ctx; + + res = pthread_create(&ti->thread, nxt_py_thread_attr, + nxt_python_thread_func, ti); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d create failed: %s (%d)", + (int) (i + 1), strerror(res), res); + } + } + + return NXT_UNIT_OK; +} + + +static void * +nxt_python_thread_func(void *data) +{ + nxt_unit_ctx_t *ctx; + PyGILState_STATE gstate; + nxt_py_thread_info_t *ti; + + ti = data; + + nxt_unit_debug(ti->ctx, "worker thread #%d start", + (int) (ti - nxt_py_threads + 1)); + + gstate = PyGILState_Ensure(); + + if (nxt_py_proto.startup != NULL) { + if (nxt_py_proto.startup(ti->ctx_data) != NXT_UNIT_OK) { + goto fail; + } + } + + ctx = nxt_unit_ctx_alloc(ti->ctx, ti->ctx_data); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + (void) nxt_py_proto.run(ctx); + + nxt_unit_done(ctx); + +fail: + + PyGILState_Release(gstate); + + nxt_unit_debug(NULL, "worker thread #%d end", + (int) (ti - nxt_py_threads + 1)); + + return NULL; +} + + +static void +nxt_python_join_threads(nxt_unit_ctx_t *ctx, nxt_python_app_conf_t *c) +{ + int res; + uint32_t i; + PyThreadState *thread_state; + nxt_py_thread_info_t *ti; + + if (nxt_py_threads == NULL) { + return; + } + + thread_state = PyEval_SaveThread(); + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + if ((uintptr_t) ti->thread == 0) { + continue; + } + + res = pthread_join(ti->thread, NULL); + + if (nxt_fast_path(res == 0)) { + nxt_unit_debug(ctx, "thread #%d joined", (int) (i + 1)); + + } else { + nxt_unit_alert(ctx, "thread #%d join failed: %s (%d)", + (int) (i + 1), strerror(res), res); + } + } + + PyEval_RestoreThread(thread_state); + + for (i = 0; i < c->threads - 1; i++) { + ti = &nxt_py_threads[i]; + + if (ti->ctx_data != NULL) { + nxt_py_proto.ctx_data_free(ti->ctx_data); + } + } + + nxt_unit_free(NULL, nxt_py_threads); +} + + +int nxt_python_init_strings(nxt_python_string_t *pstr) { PyObject *obj; @@ -271,7 +514,7 @@ nxt_python_init_strings(nxt_python_string_t *pstr) obj = PyString_FromStringAndSize((char *) pstr->string.start, pstr->string.length); if (nxt_slow_path(obj == NULL)) { - return NXT_ERROR; + return NXT_UNIT_ERROR; } PyUnicode_InternInPlace(&obj); @@ -281,7 +524,7 @@ nxt_python_init_strings(nxt_python_string_t *pstr) pstr++; } - return NXT_OK; + return NXT_UNIT_OK; } @@ -304,8 +547,9 @@ nxt_python_done_strings(nxt_python_string_t *pstr) static void nxt_python_atexit(void) { - nxt_python_wsgi_done(); - nxt_python_asgi_done(); + if (nxt_py_proto.done != NULL) { + nxt_py_proto.done(); + } Py_XDECREF(nxt_py_stderr_flush); Py_XDECREF(nxt_py_application); diff --git a/src/python/nxt_python.h b/src/python/nxt_python.h index 3211026b..b581dd46 100644 --- a/src/python/nxt_python.h +++ b/src/python/nxt_python.h @@ -11,6 +11,8 @@ #include <nxt_main.h> #include <nxt_unit.h> +#define NXT_PYTHON_VER(maj, min) ((maj << 24) | (min << 16)) + #if PY_MAJOR_VERSION == 3 #define NXT_PYTHON_BYTES_TYPE "bytestring" @@ -28,9 +30,10 @@ #define PyBytes_AS_STRING PyString_AS_STRING #define PyUnicode_InternInPlace PyString_InternInPlace #define PyUnicode_AsUTF8 PyString_AS_STRING +#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE #endif -#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 5 +#if PY_VERSION_HEX >= NXT_PYTHON_VER(3, 5) #define NXT_HAVE_ASGI 1 #endif @@ -41,20 +44,25 @@ typedef struct { PyObject **object_p; } nxt_python_string_t; +typedef struct { + int (*ctx_data_alloc)(void **pdata); + void (*ctx_data_free)(void *data); + int (*startup)(void *data); + int (*run)(nxt_unit_ctx_t *ctx); + int (*ready)(nxt_unit_ctx_t *ctx); + void (*done)(void); +} nxt_python_proto_t; + -nxt_int_t nxt_python_init_strings(nxt_python_string_t *pstr); +int nxt_python_init_strings(nxt_python_string_t *pstr); void nxt_python_done_strings(nxt_python_string_t *pstr); void nxt_python_print_exception(void); -nxt_int_t nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init); -int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx); -void nxt_python_wsgi_done(void); +int nxt_python_wsgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto); int nxt_python_asgi_check(PyObject *obj); -nxt_int_t nxt_python_asgi_init(nxt_task_t *task, nxt_unit_init_t *init); -nxt_int_t nxt_python_asgi_run(nxt_unit_ctx_t *ctx); -void nxt_python_asgi_done(void); +int nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto); #endif /* _NXT_PYTHON_H_INCLUDED_ */ diff --git a/src/python/nxt_python_asgi.c b/src/python/nxt_python_asgi.c index 72408ea1..98aeedf4 100644 --- a/src/python/nxt_python_asgi.c +++ b/src/python/nxt_python_asgi.c @@ -16,7 +16,16 @@ #include <python/nxt_python_asgi_str.h> +static PyObject *nxt_python_asgi_get_func(PyObject *obj); +static int nxt_python_asgi_ctx_data_alloc(void **pdata); +static void nxt_python_asgi_ctx_data_free(void *data); +static int nxt_python_asgi_startup(void *data); +static int nxt_python_asgi_run(nxt_unit_ctx_t *ctx); + +static void nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, + nxt_unit_port_t *port); static void nxt_py_asgi_request_handler(nxt_unit_request_info_t *req); +static void nxt_py_asgi_close_handler(nxt_unit_request_info_t *req); static PyObject *nxt_py_asgi_create_http_scope(nxt_unit_request_info_t *req); static PyObject *nxt_py_asgi_create_address(nxt_unit_sptr_t *sptr, uint8_t len, @@ -24,30 +33,33 @@ static PyObject *nxt_py_asgi_create_address(nxt_unit_sptr_t *sptr, uint8_t len, static PyObject *nxt_py_asgi_create_header(nxt_unit_field_t *f); static PyObject *nxt_py_asgi_create_subprotocols(nxt_unit_field_t *f); +static int nxt_python_asgi_ready(nxt_unit_ctx_t *ctx); + static int nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port); static void nxt_py_asgi_remove_port(nxt_unit_t *lib, nxt_unit_port_t *port); static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx); static void nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx); static PyObject *nxt_py_asgi_port_read(PyObject *self, PyObject *args); +static void nxt_python_asgi_done(void); -PyObject *nxt_py_loop_run_until_complete; -PyObject *nxt_py_loop_create_future; -PyObject *nxt_py_loop_create_task; - -nxt_queue_t nxt_py_asgi_drain_queue; - -static PyObject *nxt_py_loop_call_soon; -static PyObject *nxt_py_quit_future; -static PyObject *nxt_py_quit_future_set_result; -static PyObject *nxt_py_loop_add_reader; -static PyObject *nxt_py_loop_remove_reader; -static PyObject *nxt_py_port_read; +int nxt_py_asgi_legacy; +static PyObject *nxt_py_port_read; +static nxt_unit_port_t *nxt_py_shared_port; static PyMethodDef nxt_py_port_read_method = {"unit_port_read", nxt_py_asgi_port_read, METH_VARARGS, ""}; +static nxt_python_proto_t nxt_py_asgi_proto = { + .ctx_data_alloc = nxt_python_asgi_ctx_data_alloc, + .ctx_data_free = nxt_python_asgi_ctx_data_free, + .startup = nxt_python_asgi_startup, + .run = nxt_python_asgi_run, + .ready = nxt_python_asgi_ready, + .done = nxt_python_asgi_done, +}; + #define NXT_UNIT_HASH_WS_PROTOCOL 0xED0A @@ -55,249 +67,348 @@ int nxt_python_asgi_check(PyObject *obj) { int res; - PyObject *call; + PyObject *func; PyCodeObject *code; - if (PyFunction_Check(obj)) { - code = (PyCodeObject *) PyFunction_GET_CODE(obj); + func = nxt_python_asgi_get_func(obj); + + if (func == NULL) { + return 0; + } + + code = (PyCodeObject *) PyFunction_GET_CODE(func); + + nxt_unit_debug(NULL, "asgi_check: callable is %sa coroutine function with " + "%d argument(s)", + (code->co_flags & CO_COROUTINE) != 0 ? "" : "not ", + code->co_argcount); + + res = (code->co_flags & CO_COROUTINE) != 0 || code->co_argcount == 1; + + Py_DECREF(func); + + return res; +} + + +static PyObject * +nxt_python_asgi_get_func(PyObject *obj) +{ + PyObject *call; - return (code->co_flags & CO_COROUTINE) != 0; + if (PyFunction_Check(obj)) { + Py_INCREF(obj); + return obj; } if (PyMethod_Check(obj)) { obj = PyMethod_GET_FUNCTION(obj); - code = (PyCodeObject *) PyFunction_GET_CODE(obj); - - return (code->co_flags & CO_COROUTINE) != 0; + Py_INCREF(obj); + return obj; } call = PyObject_GetAttrString(obj, "__call__"); if (call == NULL) { - return 0; + return NULL; } if (PyFunction_Check(call)) { - code = (PyCodeObject *) PyFunction_GET_CODE(call); - - res = (code->co_flags & CO_COROUTINE) != 0; - - } else { - if (PyMethod_Check(call)) { - obj = PyMethod_GET_FUNCTION(call); + return call; + } - code = (PyCodeObject *) PyFunction_GET_CODE(obj); + if (PyMethod_Check(call)) { + obj = PyMethod_GET_FUNCTION(call); - res = (code->co_flags & CO_COROUTINE) != 0; + Py_INCREF(obj); + Py_DECREF(call); - } else { - res = 0; - } + return obj; } Py_DECREF(call); - return res; + return NULL; } -nxt_int_t -nxt_python_asgi_init(nxt_task_t *task, nxt_unit_init_t *init) +int +nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { - PyObject *asyncio, *loop, *get_event_loop; - nxt_int_t rc; + PyObject *func; + PyCodeObject *code; - nxt_debug(task, "asgi_init"); + nxt_unit_debug(NULL, "asgi_init"); - if (nxt_slow_path(nxt_py_asgi_str_init() != NXT_OK)) { - nxt_alert(task, "Python failed to init string objects"); - return NXT_ERROR; + if (nxt_slow_path(nxt_py_asgi_str_init() != NXT_UNIT_OK)) { + nxt_unit_alert(NULL, "Python failed to init string objects"); + return NXT_UNIT_ERROR; } - asyncio = PyImport_ImportModule("asyncio"); - if (nxt_slow_path(asyncio == NULL)) { - nxt_alert(task, "Python failed to import module 'asyncio'"); - nxt_python_print_exception(); - return NXT_ERROR; + nxt_py_port_read = PyCFunction_New(&nxt_py_port_read_method, NULL); + if (nxt_slow_path(nxt_py_port_read == NULL)) { + nxt_unit_alert(NULL, + "Python failed to initialize the 'port_read' function"); + return NXT_UNIT_ERROR; } - loop = NULL; - get_event_loop = PyDict_GetItemString(PyModule_GetDict(asyncio), - "get_event_loop"); - if (nxt_slow_path(get_event_loop == NULL)) { - nxt_alert(task, - "Python failed to get 'get_event_loop' from module 'asyncio'"); - goto fail; + if (nxt_slow_path(nxt_py_asgi_http_init() == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; } - if (nxt_slow_path(PyCallable_Check(get_event_loop) == 0)) { - nxt_alert(task, "'asyncio.get_event_loop' is not a callable object"); - goto fail; + if (nxt_slow_path(nxt_py_asgi_websocket_init() == NXT_UNIT_ERROR)) { + return NXT_UNIT_ERROR; } - loop = PyObject_CallObject(get_event_loop, NULL); - if (nxt_slow_path(loop == NULL)) { - nxt_alert(task, "Python failed to call 'asyncio.get_event_loop'"); - goto fail; + func = nxt_python_asgi_get_func(nxt_py_application); + if (nxt_slow_path(func == NULL)) { + nxt_unit_alert(NULL, "Python cannot find function for callable"); + return NXT_UNIT_ERROR; } - nxt_py_loop_create_task = PyObject_GetAttrString(loop, "create_task"); - if (nxt_slow_path(nxt_py_loop_create_task == NULL)) { - nxt_alert(task, "Python failed to get 'loop.create_task'"); - goto fail; - } + code = (PyCodeObject *) PyFunction_GET_CODE(func); - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_create_task) == 0)) { - nxt_alert(task, "'loop.create_task' is not a callable object"); - goto fail; + if ((code->co_flags & CO_COROUTINE) == 0) { + nxt_unit_debug(NULL, "asgi: callable is not a coroutine function " + "switching to legacy mode"); + nxt_py_asgi_legacy = 1; } - nxt_py_loop_add_reader = PyObject_GetAttrString(loop, "add_reader"); - if (nxt_slow_path(nxt_py_loop_add_reader == NULL)) { - nxt_alert(task, "Python failed to get 'loop.add_reader'"); - goto fail; - } + Py_DECREF(func); - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_add_reader) == 0)) { - nxt_alert(task, "'loop.add_reader' is not a callable object"); - goto fail; - } + init->callbacks.request_handler = nxt_py_asgi_request_handler; + init->callbacks.data_handler = nxt_py_asgi_http_data_handler; + init->callbacks.websocket_handler = nxt_py_asgi_websocket_handler; + init->callbacks.close_handler = nxt_py_asgi_close_handler; + init->callbacks.quit = nxt_py_asgi_quit; + init->callbacks.shm_ack_handler = nxt_py_asgi_shm_ack_handler; + init->callbacks.add_port = nxt_py_asgi_add_port; + init->callbacks.remove_port = nxt_py_asgi_remove_port; - nxt_py_loop_remove_reader = PyObject_GetAttrString(loop, "remove_reader"); - if (nxt_slow_path(nxt_py_loop_remove_reader == NULL)) { - nxt_alert(task, "Python failed to get 'loop.remove_reader'"); - goto fail; - } + *proto = nxt_py_asgi_proto; - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_remove_reader) == 0)) { - nxt_alert(task, "'loop.remove_reader' is not a callable object"); - goto fail; - } + return NXT_UNIT_OK; +} - nxt_py_loop_call_soon = PyObject_GetAttrString(loop, "call_soon"); - if (nxt_slow_path(nxt_py_loop_call_soon == NULL)) { - nxt_alert(task, "Python failed to get 'loop.call_soon'"); - goto fail; - } - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_call_soon) == 0)) { - nxt_alert(task, "'loop.call_soon' is not a callable object"); - goto fail; - } +static int +nxt_python_asgi_ctx_data_alloc(void **pdata) +{ + uint32_t i; + PyObject *asyncio, *loop, *new_event_loop, *obj; + nxt_py_asgi_ctx_data_t *ctx_data; - nxt_py_loop_run_until_complete = PyObject_GetAttrString(loop, - "run_until_complete"); - if (nxt_slow_path(nxt_py_loop_run_until_complete == NULL)) { - nxt_alert(task, "Python failed to get 'loop.run_until_complete'"); - goto fail; + ctx_data = nxt_unit_malloc(NULL, sizeof(nxt_py_asgi_ctx_data_t)); + if (nxt_slow_path(ctx_data == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate context data"); + return NXT_UNIT_ERROR; } - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_run_until_complete) == 0)) { - nxt_alert(task, "'loop.run_until_complete' is not a callable object"); - goto fail; - } + memset(ctx_data, 0, sizeof(nxt_py_asgi_ctx_data_t)); - nxt_py_loop_create_future = PyObject_GetAttrString(loop, "create_future"); - if (nxt_slow_path(nxt_py_loop_create_future == NULL)) { - nxt_alert(task, "Python failed to get 'loop.create_future'"); - goto fail; - } + nxt_queue_init(&ctx_data->drain_queue); - if (nxt_slow_path(PyCallable_Check(nxt_py_loop_create_future) == 0)) { - nxt_alert(task, "'loop.create_future' is not a callable object"); - goto fail; - } + struct { + const char *key; + PyObject **handler; + + } handlers[] = { + { "create_task", &ctx_data->loop_create_task }, + { "add_reader", &ctx_data->loop_add_reader }, + { "remove_reader", &ctx_data->loop_remove_reader }, + { "call_soon", &ctx_data->loop_call_soon }, + { "run_until_complete", &ctx_data->loop_run_until_complete }, + { "create_future", &ctx_data->loop_create_future }, + }; + + loop = NULL; - nxt_py_quit_future = PyObject_CallObject(nxt_py_loop_create_future, NULL); - if (nxt_slow_path(nxt_py_quit_future == NULL)) { - nxt_alert(task, "Python failed to create Future "); + asyncio = PyImport_ImportModule("asyncio"); + if (nxt_slow_path(asyncio == NULL)) { + nxt_unit_alert(NULL, "Python failed to import module 'asyncio'"); nxt_python_print_exception(); goto fail; } - nxt_py_quit_future_set_result = PyObject_GetAttrString(nxt_py_quit_future, - "set_result"); - if (nxt_slow_path(nxt_py_quit_future_set_result == NULL)) { - nxt_alert(task, "Python failed to get 'future.set_result'"); + new_event_loop = PyDict_GetItemString(PyModule_GetDict(asyncio), + "new_event_loop"); + if (nxt_slow_path(new_event_loop == NULL)) { + nxt_unit_alert(NULL, + "Python failed to get 'new_event_loop' from module 'asyncio'"); goto fail; } - if (nxt_slow_path(PyCallable_Check(nxt_py_quit_future_set_result) == 0)) { - nxt_alert(task, "'future.set_result' is not a callable object"); + if (nxt_slow_path(PyCallable_Check(new_event_loop) == 0)) { + nxt_unit_alert(NULL, + "'asyncio.new_event_loop' is not a callable object"); goto fail; } - nxt_py_port_read = PyCFunction_New(&nxt_py_port_read_method, NULL); - if (nxt_slow_path(nxt_py_port_read == NULL)) { - nxt_alert(task, "Python failed to initialize the 'port_read' function"); + loop = PyObject_CallObject(new_event_loop, NULL); + if (nxt_slow_path(loop == NULL)) { + nxt_unit_alert(NULL, "Python failed to call 'asyncio.new_event_loop'"); goto fail; } - nxt_queue_init(&nxt_py_asgi_drain_queue); + for (i = 0; i < nxt_nitems(handlers); i++) { + obj = PyObject_GetAttrString(loop, handlers[i].key); + if (nxt_slow_path(obj == NULL)) { + nxt_unit_alert(NULL, "Python failed to get 'loop.%s'", + handlers[i].key); + goto fail; + } + + *handlers[i].handler = obj; - if (nxt_slow_path(nxt_py_asgi_http_init(task) == NXT_ERROR)) { - goto fail; + if (nxt_slow_path(PyCallable_Check(obj) == 0)) { + nxt_unit_alert(NULL, "'loop.%s' is not a callable object", + handlers[i].key); + goto fail; + } } - if (nxt_slow_path(nxt_py_asgi_websocket_init(task) == NXT_ERROR)) { + obj = PyObject_CallObject(ctx_data->loop_create_future, NULL); + if (nxt_slow_path(obj == NULL)) { + nxt_unit_alert(NULL, "Python failed to create Future "); + nxt_python_print_exception(); goto fail; } - rc = nxt_py_asgi_lifespan_startup(task); - if (nxt_slow_path(rc == NXT_ERROR)) { + ctx_data->quit_future = obj; + + obj = PyObject_GetAttrString(ctx_data->quit_future, "set_result"); + if (nxt_slow_path(obj == NULL)) { + nxt_unit_alert(NULL, "Python failed to get 'future.set_result'"); goto fail; } - init->callbacks.request_handler = nxt_py_asgi_request_handler; - init->callbacks.data_handler = nxt_py_asgi_http_data_handler; - init->callbacks.websocket_handler = nxt_py_asgi_websocket_handler; - init->callbacks.close_handler = nxt_py_asgi_websocket_close_handler; - init->callbacks.quit = nxt_py_asgi_quit; - init->callbacks.shm_ack_handler = nxt_py_asgi_shm_ack_handler; - init->callbacks.add_port = nxt_py_asgi_add_port; - init->callbacks.remove_port = nxt_py_asgi_remove_port; + ctx_data->quit_future_set_result = obj; + + if (nxt_slow_path(PyCallable_Check(obj) == 0)) { + nxt_unit_alert(NULL, "'future.set_result' is not a callable object"); + goto fail; + } Py_DECREF(loop); Py_DECREF(asyncio); - return NXT_OK; + *pdata = ctx_data; + + return NXT_UNIT_OK; fail: + nxt_python_asgi_ctx_data_free(ctx_data); + Py_XDECREF(loop); - Py_DECREF(asyncio); + Py_XDECREF(asyncio); + + return NXT_UNIT_ERROR; +} + + +static void +nxt_python_asgi_ctx_data_free(void *data) +{ + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = data; - return NXT_ERROR; + Py_XDECREF(ctx_data->loop_run_until_complete); + Py_XDECREF(ctx_data->loop_create_future); + Py_XDECREF(ctx_data->loop_create_task); + Py_XDECREF(ctx_data->loop_call_soon); + Py_XDECREF(ctx_data->loop_add_reader); + Py_XDECREF(ctx_data->loop_remove_reader); + Py_XDECREF(ctx_data->quit_future); + Py_XDECREF(ctx_data->quit_future_set_result); + + nxt_unit_free(NULL, ctx_data); } -nxt_int_t +static int +nxt_python_asgi_startup(void *data) +{ + return nxt_py_asgi_lifespan_startup(data); +} + + +static int nxt_python_asgi_run(nxt_unit_ctx_t *ctx) { - PyObject *res; + PyObject *res; + nxt_py_asgi_ctx_data_t *ctx_data; - res = PyObject_CallFunctionObjArgs(nxt_py_loop_run_until_complete, - nxt_py_quit_future, NULL); + ctx_data = ctx->data; + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_run_until_complete, + ctx_data->quit_future, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to call loop.run_until_complete"); nxt_python_print_exception(); - return NXT_ERROR; + return NXT_UNIT_ERROR; } Py_DECREF(res); - nxt_py_asgi_lifespan_shutdown(); + nxt_py_asgi_remove_reader(ctx, nxt_py_shared_port); + nxt_py_asgi_remove_reader(ctx, ctx_data->port); + + if (ctx_data->port != NULL) { + ctx_data->port->data = NULL; + ctx_data->port = NULL; + } + + nxt_py_asgi_lifespan_shutdown(ctx); + + return NXT_UNIT_OK; +} + + +static void +nxt_py_asgi_remove_reader(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) +{ + PyObject *res, *fd; + nxt_py_asgi_ctx_data_t *ctx_data; + + if (port == NULL || port->in_fd == -1) { + return; + } + + ctx_data = ctx->data; + + nxt_unit_debug(ctx, "asgi_remove_reader %d %p", port->in_fd, port); + + fd = PyLong_FromLong(port->in_fd); + if (nxt_slow_path(fd == NULL)) { + nxt_unit_alert(ctx, "Python failed to create Long object"); + nxt_python_print_exception(); + + return; + } + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, fd, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to remove_reader"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } - return NXT_OK; + Py_DECREF(fd); } static void nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) { - PyObject *scope, *res, *task, *receive, *send, *done, *asgi; + PyObject *scope, *res, *task, *receive, *send, *done, *asgi; + PyObject *stage2; + nxt_py_asgi_ctx_data_t *ctx_data; if (req->request->websocket_handshake) { asgi = nxt_py_asgi_websocket_create(req); @@ -346,8 +457,42 @@ nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) req->data = asgi; - res = PyObject_CallFunctionObjArgs(nxt_py_application, - scope, receive, send, NULL); + if (!nxt_py_asgi_legacy) { + nxt_unit_req_debug(req, "Python call ASGI 3.0 application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, + scope, receive, send, NULL); + + } else { + nxt_unit_req_debug(req, "Python call legacy application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, scope, NULL); + + if (nxt_slow_path(res == NULL)) { + nxt_unit_req_error(req, "Python failed to call legacy app stage1"); + nxt_python_print_exception(); + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + goto release_scope; + } + + if (nxt_slow_path(PyCallable_Check(res) == 0)) { + nxt_unit_req_error(req, + "Legacy ASGI application returns not a callable"); + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + Py_DECREF(res); + + goto release_scope; + } + + stage2 = res; + + res = PyObject_CallFunctionObjArgs(stage2, receive, send, NULL); + + Py_DECREF(stage2); + } + if (nxt_slow_path(res == NULL)) { nxt_unit_req_error(req, "Python failed to call the application"); nxt_python_print_exception(); @@ -365,7 +510,9 @@ nxt_py_asgi_request_handler(nxt_unit_request_info_t *req) goto release_scope; } - task = PyObject_CallFunctionObjArgs(nxt_py_loop_create_task, res, NULL); + ctx_data = req->ctx->data; + + task = PyObject_CallFunctionObjArgs(ctx_data->loop_create_task, res, NULL); if (nxt_slow_path(task == NULL)) { nxt_unit_req_error(req, "Python failed to call the create_task"); nxt_python_print_exception(); @@ -405,6 +552,18 @@ release_asgi: } +static void +nxt_py_asgi_close_handler(nxt_unit_request_info_t *req) +{ + if (req->request->websocket_handshake) { + nxt_py_asgi_websocket_close_handler(req); + + } else { + nxt_py_asgi_http_close_handler(req); + } +} + + static PyObject * nxt_py_asgi_create_http_scope(nxt_unit_request_info_t *req) { @@ -724,10 +883,82 @@ fail: static int +nxt_python_asgi_ready(nxt_unit_ctx_t *ctx) +{ + int rc; + PyObject *res, *fd, *py_ctx, *py_port; + nxt_unit_port_t *port; + nxt_py_asgi_ctx_data_t *ctx_data; + + if (nxt_slow_path(nxt_py_shared_port == NULL)) { + return NXT_UNIT_ERROR; + } + + port = nxt_py_shared_port; + + nxt_unit_debug(ctx, "asgi_ready %d %p %p", port->in_fd, ctx, port); + + ctx_data = ctx->data; + + rc = NXT_UNIT_ERROR; + + fd = PyLong_FromLong(port->in_fd); + if (nxt_slow_path(fd == NULL)) { + nxt_unit_alert(ctx, "Python failed to create fd"); + nxt_python_print_exception(); + + return rc; + } + + py_ctx = PyLong_FromVoidPtr(ctx); + if (nxt_slow_path(py_ctx == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_ctx"); + nxt_python_print_exception(); + + goto clean_fd; + } + + py_port = PyLong_FromVoidPtr(port); + if (nxt_slow_path(py_port == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_port"); + nxt_python_print_exception(); + + goto clean_py_ctx; + } + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_add_reader, + fd, nxt_py_port_read, + py_ctx, py_port, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to add_reader"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + + rc = NXT_UNIT_OK; + } + + Py_DECREF(py_port); + +clean_py_ctx: + + Py_DECREF(py_ctx); + +clean_fd: + + Py_DECREF(fd); + + return rc; +} + + +static int nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) { - int nb; - PyObject *res; + int nb, rc; + PyObject *res, *fd, *py_ctx, *py_port; + nxt_py_asgi_ctx_data_t *ctx_data; if (port->in_fd == -1) { return NXT_UNIT_OK; @@ -744,73 +975,152 @@ nxt_py_asgi_add_port(nxt_unit_ctx_t *ctx, nxt_unit_port_t *port) nxt_unit_debug(ctx, "asgi_add_port %d %p %p", port->in_fd, ctx, port); - res = PyObject_CallFunctionObjArgs(nxt_py_loop_add_reader, - PyLong_FromLong(port->in_fd), - nxt_py_port_read, - PyLong_FromVoidPtr(ctx), - PyLong_FromVoidPtr(port), NULL); + if (port->id.id == NXT_UNIT_SHARED_PORT_ID) { + nxt_py_shared_port = port; + + return NXT_UNIT_OK; + } + + ctx_data = ctx->data; + + ctx_data->port = port; + port->data = ctx_data; + + rc = NXT_UNIT_ERROR; + + fd = PyLong_FromLong(port->in_fd); + if (nxt_slow_path(fd == NULL)) { + nxt_unit_alert(ctx, "Python failed to create fd"); + nxt_python_print_exception(); + + return rc; + } + + py_ctx = PyLong_FromVoidPtr(ctx); + if (nxt_slow_path(py_ctx == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_ctx"); + nxt_python_print_exception(); + + goto clean_fd; + } + + py_port = PyLong_FromVoidPtr(port); + if (nxt_slow_path(py_port == NULL)) { + nxt_unit_alert(ctx, "Python failed to create py_port"); + nxt_python_print_exception(); + + goto clean_py_ctx; + } + + res = PyObject_CallFunctionObjArgs(ctx_data->loop_add_reader, + fd, nxt_py_port_read, + py_ctx, py_port, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(ctx, "Python failed to add_reader"); + nxt_python_print_exception(); - return NXT_UNIT_ERROR; + } else { + Py_DECREF(res); + + rc = NXT_UNIT_OK; } - Py_DECREF(res); + Py_DECREF(py_port); - return NXT_UNIT_OK; +clean_py_ctx: + + Py_DECREF(py_ctx); + +clean_fd: + + Py_DECREF(fd); + + return rc; } static void nxt_py_asgi_remove_port(nxt_unit_t *lib, nxt_unit_port_t *port) { - PyObject *res; - - nxt_unit_debug(NULL, "asgi_remove_port %d %p", port->in_fd, port); - if (port->in_fd == -1) { return; } - res = PyObject_CallFunctionObjArgs(nxt_py_loop_remove_reader, - PyLong_FromLong(port->in_fd), NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(NULL, "Python failed to remove_reader"); - } + nxt_unit_debug(NULL, "asgi_remove_port %d %p", port->in_fd, port); - Py_DECREF(res); + if (nxt_py_shared_port == port) { + nxt_py_shared_port = NULL; + } } static void nxt_py_asgi_quit(nxt_unit_ctx_t *ctx) { - PyObject *res; + PyObject *res, *p; + nxt_py_asgi_ctx_data_t *ctx_data; nxt_unit_debug(ctx, "asgi_quit %p", ctx); - res = PyObject_CallFunctionObjArgs(nxt_py_quit_future_set_result, - PyLong_FromLong(0), NULL); - if (nxt_slow_path(res == NULL)) { - nxt_unit_alert(ctx, "Python failed to set_result"); + ctx_data = ctx->data; + + if (nxt_py_shared_port != NULL) { + p = PyLong_FromLong(nxt_py_shared_port->in_fd); + if (nxt_slow_path(p == NULL)) { + nxt_unit_alert(NULL, "Python failed to create Long"); + nxt_python_print_exception(); + + } else { + res = PyObject_CallFunctionObjArgs(ctx_data->loop_remove_reader, + p, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(NULL, "Python failed to remove_reader"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + + Py_DECREF(p); + } } - Py_DECREF(res); + p = PyLong_FromLong(0); + if (nxt_slow_path(p == NULL)) { + nxt_unit_alert(NULL, "Python failed to create Long"); + nxt_python_print_exception(); + + } else { + res = PyObject_CallFunctionObjArgs(ctx_data->quit_future_set_result, + p, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_alert(ctx, "Python failed to set_result"); + nxt_python_print_exception(); + + } else { + Py_DECREF(res); + } + + Py_DECREF(p); + } } static void nxt_py_asgi_shm_ack_handler(nxt_unit_ctx_t *ctx) { - int rc; - nxt_queue_link_t *lnk; + int rc; + nxt_queue_link_t *lnk; + nxt_py_asgi_ctx_data_t *ctx_data; - while (!nxt_queue_is_empty(&nxt_py_asgi_drain_queue)) { - lnk = nxt_queue_first(&nxt_py_asgi_drain_queue); + ctx_data = ctx->data; + + while (!nxt_queue_is_empty(&ctx_data->drain_queue)) { + lnk = nxt_queue_first(&ctx_data->drain_queue); rc = nxt_py_asgi_http_drain(lnk); if (rc == NXT_UNIT_AGAIN) { - break; + return; } nxt_queue_remove(lnk); @@ -859,7 +1169,7 @@ nxt_py_asgi_port_read(PyObject *self, PyObject *args) if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { return PyErr_Format(PyExc_RuntimeError, - "error processing port message"); + "error processing port %d message", port->id.id); } Py_RETURN_NONE; @@ -996,8 +1306,8 @@ nxt_py_asgi_add_field(void *data, int i, PyObject *name, PyObject *val) PyObject * -nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, - PyObject *result) +nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, + nxt_py_asgi_ctx_data_t *ctx_data, PyObject *future, PyObject *result) { PyObject *set_result, *res; @@ -1013,7 +1323,7 @@ nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, Py_CLEAR(future); - goto cleanup; + goto cleanup_result; } if (nxt_slow_path(PyCallable_Check(set_result) == 0)) { @@ -1024,7 +1334,7 @@ nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, goto cleanup; } - res = PyObject_CallFunctionObjArgs(nxt_py_loop_call_soon, set_result, + res = PyObject_CallFunctionObjArgs(ctx_data->loop_call_soon, set_result, result, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_req_alert(req, "Python failed to call 'loop.call_soon'"); @@ -1038,6 +1348,9 @@ nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, PyObject *future, cleanup: Py_DECREF(set_result); + +cleanup_result: + Py_DECREF(result); return future; @@ -1148,6 +1461,17 @@ nxt_py_asgi_new_scope(nxt_unit_request_info_t *req, PyObject *type, void +nxt_py_asgi_drain_wait(nxt_unit_request_info_t *req, nxt_queue_link_t *link) +{ + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = req->ctx->data; + + nxt_queue_insert_tail(&ctx_data->drain_queue, link); +} + + +void nxt_py_asgi_dealloc(PyObject *self) { PyObject_Del(self); @@ -1177,19 +1501,11 @@ nxt_py_asgi_next(PyObject *self) } -void +static void nxt_python_asgi_done(void) { nxt_py_asgi_str_done(); - Py_XDECREF(nxt_py_quit_future); - Py_XDECREF(nxt_py_quit_future_set_result); - Py_XDECREF(nxt_py_loop_run_until_complete); - Py_XDECREF(nxt_py_loop_create_future); - Py_XDECREF(nxt_py_loop_create_task); - Py_XDECREF(nxt_py_loop_call_soon); - Py_XDECREF(nxt_py_loop_add_reader); - Py_XDECREF(nxt_py_loop_remove_reader); Py_XDECREF(nxt_py_port_read); } @@ -1203,25 +1519,12 @@ nxt_python_asgi_check(PyObject *obj) } -nxt_int_t -nxt_python_asgi_init(nxt_task_t *task, nxt_unit_init_t *init) -{ - nxt_alert(task, "ASGI not implemented"); - return NXT_ERROR; -} - - -nxt_int_t -nxt_python_asgi_run(nxt_unit_ctx_t *ctx) +int +nxt_python_asgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { - nxt_unit_alert(ctx, "ASGI not implemented"); - return NXT_ERROR; + nxt_unit_alert(NULL, "ASGI not implemented"); + return NXT_UNIT_ERROR; } -void -nxt_python_asgi_done(void) -{ -} - #endif /* NXT_HAVE_ASGI */ diff --git a/src/python/nxt_python_asgi.h b/src/python/nxt_python_asgi.h index 24337c37..37f2a099 100644 --- a/src/python/nxt_python_asgi.h +++ b/src/python/nxt_python_asgi.h @@ -10,6 +10,9 @@ typedef PyObject * (*nxt_py_asgi_enum_header_cb)(void *ctx, int i, PyObject *name, PyObject *val); +void nxt_py_asgi_drain_wait(nxt_unit_request_info_t *req, + nxt_queue_link_t *link); + typedef struct { uint32_t fields_count; uint32_t fields_size; @@ -20,6 +23,20 @@ typedef struct { uint64_t content_length; } nxt_py_asgi_add_field_ctx_t; +typedef struct { + nxt_queue_t drain_queue; + PyObject *loop_run_until_complete; + PyObject *loop_create_future; + PyObject *loop_create_task; + PyObject *loop_call_soon; + PyObject *loop_add_reader; + PyObject *loop_remove_reader; + PyObject *quit_future; + PyObject *quit_future_set_result; + PyObject *lifespan; + nxt_unit_port_t *port; +} nxt_py_asgi_ctx_data_t; + PyObject *nxt_py_asgi_enum_headers(PyObject *headers, nxt_py_asgi_enum_header_cb cb, void *data); @@ -27,7 +44,7 @@ PyObject *nxt_py_asgi_calc_size(void *data, int i, PyObject *n, PyObject *v); PyObject *nxt_py_asgi_add_field(void *data, int i, PyObject *n, PyObject *v); PyObject *nxt_py_asgi_set_result_soon(nxt_unit_request_info_t *req, - PyObject *future, PyObject *result); + nxt_py_asgi_ctx_data_t *ctx_data, PyObject *future, PyObject *result); PyObject *nxt_py_asgi_new_msg(nxt_unit_request_info_t *req, PyObject *type); PyObject *nxt_py_asgi_new_scope(nxt_unit_request_info_t *req, PyObject *type, PyObject *spec_version); @@ -37,24 +54,21 @@ PyObject *nxt_py_asgi_await(PyObject *self); PyObject *nxt_py_asgi_iter(PyObject *self); PyObject *nxt_py_asgi_next(PyObject *self); -nxt_int_t nxt_py_asgi_http_init(nxt_task_t *task); +int nxt_py_asgi_http_init(void); PyObject *nxt_py_asgi_http_create(nxt_unit_request_info_t *req); void nxt_py_asgi_http_data_handler(nxt_unit_request_info_t *req); int nxt_py_asgi_http_drain(nxt_queue_link_t *lnk); +void nxt_py_asgi_http_close_handler(nxt_unit_request_info_t *req); -nxt_int_t nxt_py_asgi_websocket_init(nxt_task_t *task); +int nxt_py_asgi_websocket_init(void); PyObject *nxt_py_asgi_websocket_create(nxt_unit_request_info_t *req); void nxt_py_asgi_websocket_handler(nxt_unit_websocket_frame_t *ws); void nxt_py_asgi_websocket_close_handler(nxt_unit_request_info_t *req); -nxt_int_t nxt_py_asgi_lifespan_startup(nxt_task_t *task); -nxt_int_t nxt_py_asgi_lifespan_shutdown(void); - -extern PyObject *nxt_py_loop_run_until_complete; -extern PyObject *nxt_py_loop_create_future; -extern PyObject *nxt_py_loop_create_task; +int nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data); +int nxt_py_asgi_lifespan_shutdown(nxt_unit_ctx_t *ctx); -extern nxt_queue_t nxt_py_asgi_drain_queue; +extern int nxt_py_asgi_legacy; #endif /* _NXT_PYTHON_ASGI_H_INCLUDED_ */ diff --git a/src/python/nxt_python_asgi_http.c b/src/python/nxt_python_asgi_http.c index b07d61d6..d88c4b00 100644 --- a/src/python/nxt_python_asgi_http.c +++ b/src/python/nxt_python_asgi_http.c @@ -24,6 +24,7 @@ typedef struct { uint64_t content_length; uint64_t bytes_sent; int complete; + int closed; PyObject *send_body; Py_ssize_t send_body_off; } nxt_py_asgi_http_t; @@ -67,15 +68,16 @@ static PyTypeObject nxt_py_asgi_http_type = { static Py_ssize_t nxt_py_asgi_http_body_buf_size = 32 * 1024 * 1024; -nxt_int_t -nxt_py_asgi_http_init(nxt_task_t *task) +int +nxt_py_asgi_http_init(void) { if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_http_type) != 0)) { - nxt_alert(task, "Python failed to initialize the 'http' type object"); - return NXT_ERROR; + nxt_unit_alert(NULL, + "Python failed to initialize the 'http' type object"); + return NXT_UNIT_ERROR; } - return NXT_OK; + return NXT_UNIT_OK; } @@ -93,6 +95,7 @@ nxt_py_asgi_http_create(nxt_unit_request_info_t *req) http->content_length = -1; http->bytes_sent = 0; http->complete = 0; + http->closed = 0; http->send_body = NULL; http->send_body_off = 0; } @@ -106,6 +109,7 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) { PyObject *msg, *future; nxt_py_asgi_http_t *http; + nxt_py_asgi_ctx_data_t *ctx_data; nxt_unit_request_info_t *req; http = (nxt_py_asgi_http_t *) self; @@ -113,12 +117,20 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) nxt_unit_req_debug(req, "asgi_http_receive"); - msg = nxt_py_asgi_http_read_msg(http); + if (nxt_slow_path(http->closed || nxt_unit_response_is_sent(req))) { + msg = nxt_py_asgi_new_msg(req, nxt_py_http_disconnect_str); + + } else { + msg = nxt_py_asgi_http_read_msg(http); + } + if (nxt_slow_path(msg == NULL)) { return NULL; } - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + ctx_data = req->ctx->data; + + future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_req_alert(req, "Python failed to create Future object"); nxt_python_print_exception(); @@ -130,7 +142,7 @@ nxt_py_asgi_http_receive(PyObject *self, PyObject *none) } if (msg != Py_None) { - return nxt_py_asgi_set_result_soon(req, future, msg); + return nxt_py_asgi_set_result_soon(req, ctx_data, future, msg); } http->receive_future = future; @@ -250,22 +262,23 @@ nxt_py_asgi_http_send(PyObject *self, PyObject *dict) nxt_unit_req_debug(http->req, "asgi_http_send type is '%.*s'", (int) type_len, type_str); - if (type_len == (Py_ssize_t) response_start.length - && memcmp(type_str, response_start.start, type_len) == 0) - { - return nxt_py_asgi_http_response_start(http, dict); - } + if (nxt_unit_response_is_init(http->req)) { + if (nxt_str_eq(&response_body, type_str, (size_t) type_len)) { + return nxt_py_asgi_http_response_body(http, dict); + } - if (type_len == (Py_ssize_t) response_body.length - && memcmp(type_str, response_body.start, type_len) == 0) - { - return nxt_py_asgi_http_response_body(http, dict); + return PyErr_Format(PyExc_RuntimeError, + "Expected ASGI message 'http.response.body', " + "but got '%U'", type); } - nxt_unit_req_error(http->req, "asgi_http_send: unexpected 'type': '%.*s'", - (int) type_len, type_str); + if (nxt_str_eq(&response_start, type_str, (size_t) type_len)) { + return nxt_py_asgi_http_response_start(http, dict); + } - return PyErr_Format(PyExc_AssertionError, "unexpected 'type': '%U'", type); + return PyErr_Format(PyExc_RuntimeError, + "Expected ASGI message 'http.response.start', " + "but got '%U'", type); } @@ -329,11 +342,12 @@ nxt_py_asgi_http_response_start(nxt_py_asgi_http_t *http, PyObject *dict) static PyObject * nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) { - int rc; - char *body_str; - ssize_t sent; - PyObject *body, *more_body, *future; - Py_ssize_t body_len, body_off; + int rc; + char *body_str; + ssize_t sent; + PyObject *body, *more_body, *future; + Py_ssize_t body_len, body_off; + nxt_py_asgi_ctx_data_t *ctx_data; body = PyDict_GetItem(dict, nxt_py_body_str); if (nxt_slow_path(body != NULL && !PyBytes_Check(body))) { @@ -371,6 +385,8 @@ nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) body_off = 0; + ctx_data = http->req->ctx->data; + while (body_len > 0) { sent = nxt_unit_response_write_nb(http->req, body_str, body_len, 0); if (nxt_slow_path(sent < 0)) { @@ -382,7 +398,8 @@ nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) "out of shared memory, %d", (int) body_len); - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + future = PyObject_CallObject(ctx_data->loop_create_future, + NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_req_alert(http->req, "Python failed to create Future object"); @@ -396,7 +413,7 @@ nxt_py_asgi_http_response_body(nxt_py_asgi_http_t *http, PyObject *dict) Py_INCREF(http->send_body); http->send_body_off = body_off; - nxt_queue_insert_tail(&nxt_py_asgi_drain_queue, &http->link); + nxt_py_asgi_drain_wait(http->req, &http->link); http->send_future = future; Py_INCREF(http->send_future); @@ -553,6 +570,48 @@ fail: } +void +nxt_py_asgi_http_close_handler(nxt_unit_request_info_t *req) +{ + PyObject *msg, *future, *res; + nxt_py_asgi_http_t *http; + + http = req->data; + + nxt_unit_req_debug(req, "asgi_http_close_handler"); + + http->closed = 1; + + if (http->receive_future == NULL) { + return; + } + + msg = nxt_py_asgi_new_msg(req, nxt_py_http_disconnect_str); + if (nxt_slow_path(msg == NULL)) { + return; + } + + if (msg == Py_None) { + Py_DECREF(msg); + return; + } + + future = http->receive_future; + http->receive_future = NULL; + + res = PyObject_CallMethodObjArgs(future, nxt_py_set_result_str, msg, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_req_alert(req, "'set_result' call failed"); + nxt_python_print_exception(); + } + + Py_XDECREF(res); + Py_DECREF(future); + + Py_DECREF(msg); +} + + static PyObject * nxt_py_asgi_http_done(PyObject *self, PyObject *future) { diff --git a/src/python/nxt_python_asgi_lifespan.c b/src/python/nxt_python_asgi_lifespan.c index 14d0ee97..506eaf4d 100644 --- a/src/python/nxt_python_asgi_lifespan.c +++ b/src/python/nxt_python_asgi_lifespan.c @@ -15,15 +15,16 @@ typedef struct { PyObject_HEAD - int disabled; - int startup_received; - int startup_sent; - int shutdown_received; - int shutdown_sent; - int shutdown_called; - PyObject *startup_future; - PyObject *shutdown_future; - PyObject *receive_future; + nxt_py_asgi_ctx_data_t *ctx_data; + int disabled; + int startup_received; + int startup_sent; + int shutdown_received; + int shutdown_sent; + int shutdown_called; + PyObject *startup_future; + PyObject *shutdown_future; + PyObject *receive_future; } nxt_py_asgi_lifespan_t; @@ -39,8 +40,6 @@ static PyObject *nxt_py_asgi_lifespan_disable(nxt_py_asgi_lifespan_t *lifespan); static PyObject *nxt_py_asgi_lifespan_done(PyObject *self, PyObject *future); -static nxt_py_asgi_lifespan_t *nxt_py_lifespan; - static PyMethodDef nxt_py_asgi_lifespan_methods[] = { { "receive", nxt_py_asgi_lifespan_receive, METH_NOARGS, 0 }, { "send", nxt_py_asgi_lifespan_send, METH_O, 0 }, @@ -67,46 +66,47 @@ static PyTypeObject nxt_py_asgi_lifespan_type = { }; -nxt_int_t -nxt_py_asgi_lifespan_startup(nxt_task_t *task) +int +nxt_py_asgi_lifespan_startup(nxt_py_asgi_ctx_data_t *ctx_data) { + int rc; PyObject *scope, *res, *py_task, *receive, *send, *done; - nxt_int_t rc; + PyObject *stage2; nxt_py_asgi_lifespan_t *lifespan; if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_lifespan_type) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to initialize the 'asgi_lifespan' type object"); - return NXT_ERROR; + return NXT_UNIT_ERROR; } lifespan = PyObject_New(nxt_py_asgi_lifespan_t, &nxt_py_asgi_lifespan_type); if (nxt_slow_path(lifespan == NULL)) { - nxt_alert(task, "Python failed to create lifespan object"); - return NXT_ERROR; + nxt_unit_alert(NULL, "Python failed to create lifespan object"); + return NXT_UNIT_ERROR; } - rc = NXT_ERROR; + rc = NXT_UNIT_ERROR; receive = PyObject_GetAttrString((PyObject *) lifespan, "receive"); if (nxt_slow_path(receive == NULL)) { - nxt_alert(task, "Python failed to get 'receive' method"); + nxt_unit_alert(NULL, "Python failed to get 'receive' method"); goto release_lifespan; } send = PyObject_GetAttrString((PyObject *) lifespan, "send"); if (nxt_slow_path(receive == NULL)) { - nxt_alert(task, "Python failed to get 'send' method"); + nxt_unit_alert(NULL, "Python failed to get 'send' method"); goto release_receive; } done = PyObject_GetAttrString((PyObject *) lifespan, "_done"); if (nxt_slow_path(receive == NULL)) { - nxt_alert(task, "Python failed to get '_done' method"); + nxt_unit_alert(NULL, "Python failed to get '_done' method"); goto release_send; } - lifespan->startup_future = PyObject_CallObject(nxt_py_loop_create_future, + lifespan->startup_future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(lifespan->startup_future == NULL)) { nxt_unit_alert(NULL, "Python failed to create Future object"); @@ -115,6 +115,7 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) goto release_done; } + lifespan->ctx_data = ctx_data; lifespan->disabled = 0; lifespan->startup_received = 0; lifespan->startup_sent = 0; @@ -129,24 +130,59 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) goto release_future; } - res = PyObject_CallFunctionObjArgs(nxt_py_application, - scope, receive, send, NULL); + if (!nxt_py_asgi_legacy) { + nxt_unit_req_debug(NULL, "Python call ASGI 3.0 application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, + scope, receive, send, NULL); + + } else { + nxt_unit_req_debug(NULL, "Python call legacy application"); + + res = PyObject_CallFunctionObjArgs(nxt_py_application, scope, NULL); + if (nxt_slow_path(res == NULL)) { + nxt_unit_log(NULL, NXT_UNIT_LOG_INFO, + "ASGI Lifespan processing exception"); + nxt_python_print_exception(); + + lifespan->disabled = 1; + rc = NXT_UNIT_OK; + + goto release_scope; + } + + if (nxt_slow_path(PyCallable_Check(res) == 0)) { + nxt_unit_req_error(NULL, + "Legacy ASGI application returns not a callable"); + + Py_DECREF(res); + + goto release_scope; + } + + stage2 = res; + + res = PyObject_CallFunctionObjArgs(stage2, receive, send, NULL); + + Py_DECREF(stage2); + } + if (nxt_slow_path(res == NULL)) { - nxt_log(task, NXT_LOG_ERR, "Python failed to call the application"); + nxt_unit_error(NULL, "Python failed to call the application"); nxt_python_print_exception(); goto release_scope; } if (nxt_slow_path(!PyCoro_CheckExact(res))) { - nxt_log(task, NXT_LOG_ERR, - "Application result type is not a coroutine"); + nxt_unit_error(NULL, "Application result type is not a coroutine"); Py_DECREF(res); goto release_scope; } - py_task = PyObject_CallFunctionObjArgs(nxt_py_loop_create_task, res, NULL); + py_task = PyObject_CallFunctionObjArgs(ctx_data->loop_create_task, res, + NULL); if (nxt_slow_path(py_task == NULL)) { - nxt_log(task, NXT_LOG_ERR, "Python failed to call the create_task"); + nxt_unit_alert(NULL, "Python failed to call the create_task"); nxt_python_print_exception(); Py_DECREF(res); goto release_scope; @@ -157,18 +193,17 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) res = PyObject_CallMethodObjArgs(py_task, nxt_py_add_done_callback_str, done, NULL); if (nxt_slow_path(res == NULL)) { - nxt_log(task, NXT_LOG_ERR, - "Python failed to call 'task.add_done_callback'"); + nxt_unit_alert(NULL, "Python failed to call 'task.add_done_callback'"); nxt_python_print_exception(); goto release_task; } Py_DECREF(res); - res = PyObject_CallFunctionObjArgs(nxt_py_loop_run_until_complete, + res = PyObject_CallFunctionObjArgs(ctx_data->loop_run_until_complete, lifespan->startup_future, NULL); if (nxt_slow_path(res == NULL)) { - nxt_alert(task, "Python failed to call loop.run_until_complete"); + nxt_unit_alert(NULL, "Python failed to call loop.run_until_complete"); nxt_python_print_exception(); goto release_task; } @@ -176,10 +211,10 @@ nxt_py_asgi_lifespan_startup(nxt_task_t *task) Py_DECREF(res); if (lifespan->startup_sent == 1 || lifespan->disabled) { - nxt_py_lifespan = lifespan; - Py_INCREF(nxt_py_lifespan); + ctx_data->lifespan = (PyObject *) lifespan; + Py_INCREF(ctx_data->lifespan); - rc = NXT_OK; + rc = NXT_UNIT_OK; } release_task: @@ -201,17 +236,21 @@ release_lifespan: } -nxt_int_t -nxt_py_asgi_lifespan_shutdown(void) +int +nxt_py_asgi_lifespan_shutdown(nxt_unit_ctx_t *ctx) { PyObject *msg, *future, *res; nxt_py_asgi_lifespan_t *lifespan; + nxt_py_asgi_ctx_data_t *ctx_data; + + ctx_data = ctx->data; + + lifespan = (nxt_py_asgi_lifespan_t *) ctx_data->lifespan; - if (nxt_slow_path(nxt_py_lifespan == NULL || nxt_py_lifespan->disabled)) { - return NXT_OK; + if (nxt_slow_path(lifespan == NULL || lifespan->disabled)) { + return NXT_UNIT_OK; } - lifespan = nxt_py_lifespan; lifespan->shutdown_called = 1; if (lifespan->receive_future != NULL) { @@ -231,29 +270,29 @@ nxt_py_asgi_lifespan_shutdown(void) } if (lifespan->shutdown_sent) { - return NXT_OK; + return NXT_UNIT_OK; } - lifespan->shutdown_future = PyObject_CallObject(nxt_py_loop_create_future, + lifespan->shutdown_future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(lifespan->shutdown_future == NULL)) { nxt_unit_alert(NULL, "Python failed to create Future object"); nxt_python_print_exception(); - return NXT_ERROR; + return NXT_UNIT_ERROR; } - res = PyObject_CallFunctionObjArgs(nxt_py_loop_run_until_complete, + res = PyObject_CallFunctionObjArgs(ctx_data->loop_run_until_complete, lifespan->shutdown_future, NULL); if (nxt_slow_path(res == NULL)) { nxt_unit_alert(NULL, "Python failed to call loop.run_until_complete"); nxt_python_print_exception(); - return NXT_ERROR; + return NXT_UNIT_ERROR; } Py_DECREF(res); Py_CLEAR(lifespan->shutdown_future); - return NXT_OK; + return NXT_UNIT_OK; } @@ -262,12 +301,14 @@ nxt_py_asgi_lifespan_receive(PyObject *self, PyObject *none) { PyObject *msg, *future; nxt_py_asgi_lifespan_t *lifespan; + nxt_py_asgi_ctx_data_t *ctx_data; lifespan = (nxt_py_asgi_lifespan_t *) self; + ctx_data = lifespan->ctx_data; nxt_unit_debug(NULL, "asgi_lifespan_receive"); - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_alert(NULL, "Python failed to create Future object"); nxt_python_print_exception(); @@ -281,7 +322,7 @@ nxt_py_asgi_lifespan_receive(PyObject *self, PyObject *none) msg = nxt_py_asgi_new_msg(NULL, nxt_py_lifespan_startup_str); - return nxt_py_asgi_set_result_soon(NULL, future, msg); + return nxt_py_asgi_set_result_soon(NULL, ctx_data, future, msg); } if (lifespan->shutdown_called && !lifespan->shutdown_received) { @@ -289,7 +330,7 @@ nxt_py_asgi_lifespan_receive(PyObject *self, PyObject *none) msg = nxt_py_asgi_new_msg(NULL, nxt_py_lifespan_shutdown_str); - return nxt_py_asgi_set_result_soon(NULL, future, msg); + return nxt_py_asgi_set_result_soon(NULL, ctx_data, future, msg); } Py_INCREF(future); diff --git a/src/python/nxt_python_asgi_str.c b/src/python/nxt_python_asgi_str.c index 37fa7f04..34422973 100644 --- a/src/python/nxt_python_asgi_str.c +++ b/src/python/nxt_python_asgi_str.c @@ -124,7 +124,7 @@ static nxt_python_string_t nxt_py_asgi_strings[] = { }; -nxt_int_t +int nxt_py_asgi_str_init(void) { return nxt_python_init_strings(nxt_py_asgi_strings); diff --git a/src/python/nxt_python_asgi_str.h b/src/python/nxt_python_asgi_str.h index 3f389c62..92969fd2 100644 --- a/src/python/nxt_python_asgi_str.h +++ b/src/python/nxt_python_asgi_str.h @@ -62,7 +62,7 @@ extern PyObject *nxt_py_ws_str; extern PyObject *nxt_py_wss_str; -nxt_int_t nxt_py_asgi_str_init(void); +int nxt_py_asgi_str_init(void); void nxt_py_asgi_str_done(void); diff --git a/src/python/nxt_python_asgi_websocket.c b/src/python/nxt_python_asgi_websocket.c index 5a27b588..fc7d9fa4 100644 --- a/src/python/nxt_python_asgi_websocket.c +++ b/src/python/nxt_python_asgi_websocket.c @@ -98,16 +98,16 @@ static uint64_t nxt_py_asgi_ws_max_frame_size = 1024 * 1024; static uint64_t nxt_py_asgi_ws_max_buffer_size = 10 * 1024 * 1024; -nxt_int_t -nxt_py_asgi_websocket_init(nxt_task_t *task) +int +nxt_py_asgi_websocket_init(void) { if (nxt_slow_path(PyType_Ready(&nxt_py_asgi_websocket_type) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to initialize the \"asgi_websocket\" type object"); - return NXT_ERROR; + return NXT_UNIT_ERROR; } - return NXT_OK; + return NXT_UNIT_OK; } @@ -137,6 +137,7 @@ static PyObject * nxt_py_asgi_websocket_receive(PyObject *self, PyObject *none) { PyObject *future, *msg; + nxt_py_asgi_ctx_data_t *ctx_data; nxt_py_asgi_websocket_t *ws; ws = (nxt_py_asgi_websocket_t *) self; @@ -160,7 +161,9 @@ nxt_py_asgi_websocket_receive(PyObject *self, PyObject *none) "WebSocket already closed"); } - future = PyObject_CallObject(nxt_py_loop_create_future, NULL); + ctx_data = ws->req->ctx->data; + + future = PyObject_CallObject(ctx_data->loop_create_future, NULL); if (nxt_slow_path(future == NULL)) { nxt_unit_req_alert(ws->req, "Python failed to create Future object"); nxt_python_print_exception(); @@ -174,19 +177,19 @@ nxt_py_asgi_websocket_receive(PyObject *self, PyObject *none) msg = nxt_py_asgi_new_msg(ws->req, nxt_py_websocket_connect_str); - return nxt_py_asgi_set_result_soon(ws->req, future, msg); + return nxt_py_asgi_set_result_soon(ws->req, ctx_data, future, msg); } if (ws->pending_fins > 0) { msg = nxt_py_asgi_websocket_pop_msg(ws, NULL); - return nxt_py_asgi_set_result_soon(ws->req, future, msg); + return nxt_py_asgi_set_result_soon(ws->req, ctx_data, future, msg); } if (nxt_slow_path(ws->state == NXT_WS_DISCONNECTED)) { msg = nxt_py_asgi_websocket_disconnect_msg(ws); - return nxt_py_asgi_set_result_soon(ws->req, future, msg); + return nxt_py_asgi_set_result_soon(ws->req, ctx_data, future, msg); } ws->receive_future = future; diff --git a/src/python/nxt_python_wsgi.c b/src/python/nxt_python_wsgi.c index 97030cd3..da7b183c 100644 --- a/src/python/nxt_python_wsgi.c +++ b/src/python/nxt_python_wsgi.c @@ -38,55 +38,57 @@ */ -typedef struct nxt_python_run_ctx_s nxt_python_run_ctx_t; - typedef struct { PyObject_HEAD -} nxt_py_input_t; + uint64_t content_length; + uint64_t bytes_sent; + PyObject *environ; + PyObject *start_resp; + PyObject *write; + nxt_unit_request_info_t *req; + PyThreadState *thread_state; +} nxt_python_ctx_t; -typedef struct { - PyObject_HEAD -} nxt_py_error_t; + +static int nxt_python_wsgi_ctx_data_alloc(void **pdata); +static void nxt_python_wsgi_ctx_data_free(void *data); +static int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx); +static void nxt_python_wsgi_done(void); static void nxt_python_request_handler(nxt_unit_request_info_t *req); -static PyObject *nxt_python_create_environ(nxt_task_t *task); -static PyObject *nxt_python_get_environ(nxt_python_run_ctx_t *ctx); -static int nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, +static PyObject *nxt_python_create_environ(nxt_python_app_conf_t *c); +static PyObject *nxt_python_get_environ(nxt_python_ctx_t *pctx); +static int nxt_python_add_sptr(nxt_python_ctx_t *pctx, PyObject *name, nxt_unit_sptr_t *sptr, uint32_t size); -static int nxt_python_add_field(nxt_python_run_ctx_t *ctx, +static int nxt_python_add_field(nxt_python_ctx_t *pctx, nxt_unit_field_t *field, int n, uint32_t vl); static PyObject *nxt_python_field_name(const char *name, uint8_t len); static PyObject *nxt_python_field_value(nxt_unit_field_t *f, int n, uint32_t vl); -static int nxt_python_add_obj(nxt_python_run_ctx_t *ctx, PyObject *name, +static int nxt_python_add_obj(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value); static PyObject *nxt_py_start_resp(PyObject *self, PyObject *args); -static int nxt_python_response_add_field(nxt_python_run_ctx_t *ctx, +static int nxt_python_response_add_field(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value, int i); static int nxt_python_str_buf(PyObject *str, char **buf, uint32_t *len, PyObject **bytes); static PyObject *nxt_py_write(PyObject *self, PyObject *args); -static void nxt_py_input_dealloc(nxt_py_input_t *self); -static PyObject *nxt_py_input_read(nxt_py_input_t *self, PyObject *args); -static PyObject *nxt_py_input_readline(nxt_py_input_t *self, PyObject *args); -static PyObject *nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size); -static PyObject *nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args); +static void nxt_py_input_dealloc(nxt_python_ctx_t *pctx); +static PyObject *nxt_py_input_read(nxt_python_ctx_t *pctx, PyObject *args); +static PyObject *nxt_py_input_readline(nxt_python_ctx_t *pctx, + PyObject *args); +static PyObject *nxt_py_input_getline(nxt_python_ctx_t *pctx, size_t size); +static PyObject *nxt_py_input_readlines(nxt_python_ctx_t *self, + PyObject *args); -static PyObject *nxt_py_input_iter(PyObject *self); -static PyObject *nxt_py_input_next(PyObject *self); +static PyObject *nxt_py_input_iter(PyObject *pctx); +static PyObject *nxt_py_input_next(PyObject *pctx); -static int nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes); - -struct nxt_python_run_ctx_s { - uint64_t content_length; - uint64_t bytes_sent; - PyObject *environ; - nxt_unit_request_info_t *req; -}; +static int nxt_python_write(nxt_python_ctx_t *pctx, PyObject *bytes); static PyMethodDef nxt_py_start_resp_method[] = { @@ -111,7 +113,7 @@ static PyTypeObject nxt_py_input_type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "unit._input", - .tp_basicsize = sizeof(nxt_py_input_t), + .tp_basicsize = sizeof(nxt_python_ctx_t), .tp_dealloc = (destructor) nxt_py_input_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_doc = "unit input object.", @@ -121,12 +123,7 @@ static PyTypeObject nxt_py_input_type = { }; -static PyObject *nxt_py_start_resp_obj; -static PyObject *nxt_py_write_obj; -static PyObject *nxt_py_environ_ptyp; - -static PyThreadState *nxt_python_thread_state; -static nxt_python_run_ctx_t *nxt_python_run_ctx; +static PyObject *nxt_py_environ_ptyp; static PyObject *nxt_py_80_str; static PyObject *nxt_py_close_str; @@ -143,6 +140,7 @@ static PyObject *nxt_py_server_addr_str; static PyObject *nxt_py_server_name_str; static PyObject *nxt_py_server_port_str; static PyObject *nxt_py_server_protocol_str; +static PyObject *nxt_py_wsgi_input_str; static PyObject *nxt_py_wsgi_uri_scheme_str; static nxt_python_string_t nxt_python_strings[] = { @@ -161,82 +159,132 @@ static nxt_python_string_t nxt_python_strings[] = { { nxt_string("SERVER_NAME"), &nxt_py_server_name_str }, { nxt_string("SERVER_PORT"), &nxt_py_server_port_str }, { nxt_string("SERVER_PROTOCOL"), &nxt_py_server_protocol_str }, + { nxt_string("wsgi.input"), &nxt_py_wsgi_input_str }, { nxt_string("wsgi.url_scheme"), &nxt_py_wsgi_uri_scheme_str }, { nxt_null_string, NULL }, }; +static nxt_python_proto_t nxt_py_wsgi_proto = { + .ctx_data_alloc = nxt_python_wsgi_ctx_data_alloc, + .ctx_data_free = nxt_python_wsgi_ctx_data_free, + .run = nxt_python_wsgi_run, + .done = nxt_python_wsgi_done, +}; + -nxt_int_t -nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init) +int +nxt_python_wsgi_init(nxt_unit_init_t *init, nxt_python_proto_t *proto) { PyObject *obj; obj = NULL; - if (nxt_slow_path(nxt_python_init_strings(nxt_python_strings) != NXT_OK)) { - nxt_alert(task, "Python failed to init string objects"); + if (nxt_slow_path(nxt_python_init_strings(nxt_python_strings) + != NXT_UNIT_OK)) + { + nxt_unit_alert(NULL, "Python failed to init string objects"); goto fail; } - obj = PyCFunction_New(nxt_py_start_resp_method, NULL); + obj = nxt_python_create_environ(init->data); if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, - "Python failed to initialize the \"start_response\" function"); goto fail; } - nxt_py_start_resp_obj = obj; + nxt_py_environ_ptyp = obj; + obj = NULL; + + init->callbacks.request_handler = nxt_python_request_handler; + + *proto = nxt_py_wsgi_proto; - obj = PyCFunction_New(nxt_py_write_method, NULL); - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to initialize the \"write\" function"); - goto fail; + return NXT_UNIT_OK; + +fail: + + Py_XDECREF(obj); + + return NXT_UNIT_ERROR; +} + + +static int +nxt_python_wsgi_ctx_data_alloc(void **pdata) +{ + nxt_python_ctx_t *pctx; + + pctx = PyObject_New(nxt_python_ctx_t, &nxt_py_input_type); + if (nxt_slow_path(pctx == NULL)) { + nxt_unit_alert(NULL, + "Python failed to create the \"wsgi.input\" object"); + return NXT_UNIT_ERROR; } - nxt_py_write_obj = obj; + pctx->write = NULL; - obj = nxt_python_create_environ(task); - if (nxt_slow_path(obj == NULL)) { + pctx->start_resp = PyCFunction_New(nxt_py_start_resp_method, + (PyObject *) pctx); + if (nxt_slow_path(pctx->start_resp == NULL)) { + nxt_unit_alert(NULL, + "Python failed to initialize the \"start_response\" function"); goto fail; } - nxt_py_environ_ptyp = obj; - obj = NULL; + pctx->write = PyCFunction_New(nxt_py_write_method, (PyObject *) pctx); + if (nxt_slow_path(pctx->write == NULL)) { + nxt_unit_alert(NULL, + "Python failed to initialize the \"write\" function"); + goto fail; + } - init->callbacks.request_handler = nxt_python_request_handler; + *pdata = pctx; - return NXT_OK; + return NXT_UNIT_OK; fail: - Py_XDECREF(obj); + nxt_python_wsgi_ctx_data_free(pctx); - return NXT_ERROR; + return NXT_UNIT_ERROR; } -int +static void +nxt_python_wsgi_ctx_data_free(void *data) +{ + nxt_python_ctx_t *pctx; + + pctx = data; + + Py_XDECREF(pctx->start_resp); + Py_XDECREF(pctx->write); + Py_XDECREF(pctx); +} + + +static int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx) { - int rc; + int rc; + nxt_python_ctx_t *pctx; - nxt_python_thread_state = PyEval_SaveThread(); + pctx = ctx->data; + + pctx->thread_state = PyEval_SaveThread(); rc = nxt_unit_run(ctx); - PyEval_RestoreThread(nxt_python_thread_state); + PyEval_RestoreThread(pctx->thread_state); return rc; } -void +static void nxt_python_wsgi_done(void) { nxt_python_done_strings(nxt_python_strings); - Py_XDECREF(nxt_py_start_resp_obj); - Py_XDECREF(nxt_py_write_obj); Py_XDECREF(nxt_py_environ_ptyp); } @@ -244,14 +292,20 @@ nxt_python_wsgi_done(void) static void nxt_python_request_handler(nxt_unit_request_info_t *req) { - int rc; - PyObject *environ, *args, *response, *iterator, *item; - PyObject *close, *result; - nxt_python_run_ctx_t run_ctx = {-1, 0, NULL, req}; + int rc; + PyObject *environ, *args, *response, *iterator, *item; + PyObject *close, *result; + nxt_python_ctx_t *pctx; - PyEval_RestoreThread(nxt_python_thread_state); + pctx = req->ctx->data; - environ = nxt_python_get_environ(&run_ctx); + pctx->content_length = -1; + pctx->bytes_sent = 0; + pctx->req = req; + + PyEval_RestoreThread(pctx->thread_state); + + environ = nxt_python_get_environ(pctx); if (nxt_slow_path(environ == NULL)) { rc = NXT_UNIT_ERROR; goto done; @@ -269,10 +323,8 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) PyTuple_SET_ITEM(args, 0, environ); - Py_INCREF(nxt_py_start_resp_obj); - PyTuple_SET_ITEM(args, 1, nxt_py_start_resp_obj); - - nxt_python_run_ctx = &run_ctx; + Py_INCREF(pctx->start_resp); + PyTuple_SET_ITEM(args, 1, pctx->start_resp); response = PyObject_CallObject(nxt_py_application, args); @@ -288,7 +340,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) /* Shortcut: avoid iterate over response string symbols. */ if (PyBytes_Check(response)) { - rc = nxt_python_write(&run_ctx, response); + rc = nxt_python_write(pctx, response); } else { iterator = PyObject_GetIter(response); @@ -296,7 +348,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) if (nxt_fast_path(iterator != NULL)) { rc = NXT_UNIT_OK; - while (run_ctx.bytes_sent < run_ctx.content_length) { + while (pctx->bytes_sent < pctx->content_length) { item = PyIter_Next(iterator); if (item == NULL) { @@ -312,7 +364,7 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) } if (nxt_fast_path(PyBytes_Check(item))) { - rc = nxt_python_write(&run_ctx, item); + rc = nxt_python_write(pctx, item); } else { nxt_unit_req_error(req, "the application returned " @@ -361,29 +413,31 @@ nxt_python_request_handler(nxt_unit_request_info_t *req) done: - nxt_python_thread_state = PyEval_SaveThread(); + pctx->thread_state = PyEval_SaveThread(); + + pctx->req = NULL; - nxt_python_run_ctx = NULL; nxt_unit_request_done(req, rc); } static PyObject * -nxt_python_create_environ(nxt_task_t *task) +nxt_python_create_environ(nxt_python_app_conf_t *c) { PyObject *obj, *err, *environ; environ = PyDict_New(); if (nxt_slow_path(environ == NULL)) { - nxt_alert(task, "Python failed to create the \"environ\" dictionary"); + nxt_unit_alert(NULL, + "Python failed to create the \"environ\" dictionary"); return NULL; } obj = PyString_FromStringAndSize((char *) nxt_server.start, nxt_server.length); if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to create the \"SERVER_SOFTWARE\" environ value"); goto fail; } @@ -391,7 +445,7 @@ nxt_python_create_environ(nxt_task_t *task) if (nxt_slow_path(PyDict_SetItemString(environ, "SERVER_SOFTWARE", obj) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"SERVER_SOFTWARE\" environ value"); goto fail; } @@ -401,15 +455,15 @@ nxt_python_create_environ(nxt_task_t *task) obj = Py_BuildValue("(ii)", 1, 0); if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to build the \"wsgi.version\" environ value"); goto fail; } if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.version", obj) != 0)) { - nxt_alert(task, - "Python failed to set the \"wsgi.version\" environ value"); + nxt_unit_alert(NULL, + "Python failed to set the \"wsgi.version\" environ value"); goto fail; } @@ -418,10 +472,10 @@ nxt_python_create_environ(nxt_task_t *task) if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.multithread", - Py_False) + c->threads > 1 ? Py_True : Py_False) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"wsgi.multithread\" environ value"); goto fail; } @@ -430,7 +484,7 @@ nxt_python_create_environ(nxt_task_t *task) Py_True) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"wsgi.multiprocess\" environ value"); goto fail; } @@ -439,46 +493,30 @@ nxt_python_create_environ(nxt_task_t *task) Py_False) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to set the \"wsgi.run_once\" environ value"); goto fail; } if (nxt_slow_path(PyType_Ready(&nxt_py_input_type) != 0)) { - nxt_alert(task, + nxt_unit_alert(NULL, "Python failed to initialize the \"wsgi.input\" type object"); goto fail; } - obj = (PyObject *) PyObject_New(nxt_py_input_t, &nxt_py_input_type); - - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to create the \"wsgi.input\" object"); - goto fail; - } - - if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.input", obj) != 0)) { - nxt_alert(task, - "Python failed to set the \"wsgi.input\" environ value"); - goto fail; - } - - Py_DECREF(obj); - obj = NULL; - err = PySys_GetObject((char *) "stderr"); if (nxt_slow_path(err == NULL)) { - nxt_alert(task, "Python failed to get \"sys.stderr\" object"); + nxt_unit_alert(NULL, "Python failed to get \"sys.stderr\" object"); goto fail; } if (nxt_slow_path(PyDict_SetItemString(environ, "wsgi.errors", err) != 0)) { - nxt_alert(task, - "Python failed to set the \"wsgi.errors\" environ value"); + nxt_unit_alert(NULL, + "Python failed to set the \"wsgi.errors\" environ value"); goto fail; } @@ -494,7 +532,7 @@ fail: static PyObject * -nxt_python_get_environ(nxt_python_run_ctx_t *ctx) +nxt_python_get_environ(nxt_python_ctx_t *pctx) { int rc; uint32_t i, j, vl; @@ -504,15 +542,15 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) environ = PyDict_Copy(nxt_py_environ_ptyp); if (nxt_slow_path(environ == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to copy the \"environ\" dictionary"); return NULL; } - ctx->environ = environ; + pctx->environ = environ; - r = ctx->req->request; + r = pctx->req->request; #define RC(S) \ do { \ @@ -522,36 +560,36 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) } \ } while(0) - RC(nxt_python_add_sptr(ctx, nxt_py_request_method_str, &r->method, + RC(nxt_python_add_sptr(pctx, nxt_py_request_method_str, &r->method, r->method_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_request_uri_str, &r->target, + RC(nxt_python_add_sptr(pctx, nxt_py_request_uri_str, &r->target, r->target_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_query_string_str, &r->query, + RC(nxt_python_add_sptr(pctx, nxt_py_query_string_str, &r->query, r->query_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_path_info_str, &r->path, + RC(nxt_python_add_sptr(pctx, nxt_py_path_info_str, &r->path, r->path_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_remote_addr_str, &r->remote, + RC(nxt_python_add_sptr(pctx, nxt_py_remote_addr_str, &r->remote, r->remote_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_server_addr_str, &r->local, + RC(nxt_python_add_sptr(pctx, nxt_py_server_addr_str, &r->local, r->local_length)); if (r->tls) { - RC(nxt_python_add_obj(ctx, nxt_py_wsgi_uri_scheme_str, + RC(nxt_python_add_obj(pctx, nxt_py_wsgi_uri_scheme_str, nxt_py_https_str)); } else { - RC(nxt_python_add_obj(ctx, nxt_py_wsgi_uri_scheme_str, + RC(nxt_python_add_obj(pctx, nxt_py_wsgi_uri_scheme_str, nxt_py_http_str)); } - RC(nxt_python_add_sptr(ctx, nxt_py_server_protocol_str, &r->version, + RC(nxt_python_add_sptr(pctx, nxt_py_server_protocol_str, &r->version, r->version_length)); - RC(nxt_python_add_sptr(ctx, nxt_py_server_name_str, &r->server_name, + RC(nxt_python_add_sptr(pctx, nxt_py_server_name_str, &r->server_name, r->server_name_length)); - RC(nxt_python_add_obj(ctx, nxt_py_server_port_str, nxt_py_80_str)); + RC(nxt_python_add_obj(pctx, nxt_py_server_port_str, nxt_py_80_str)); - nxt_unit_request_group_dup_fields(ctx->req); + nxt_unit_request_group_dup_fields(pctx->req); for (i = 0; i < r->fields_count;) { f = r->fields + i; @@ -569,7 +607,7 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) vl += 2 + f2->value_length; } - RC(nxt_python_add_field(ctx, f, j - i, vl)); + RC(nxt_python_add_field(pctx, f, j - i, vl)); i = j; } @@ -577,19 +615,27 @@ nxt_python_get_environ(nxt_python_run_ctx_t *ctx) if (r->content_length_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_length_field; - RC(nxt_python_add_sptr(ctx, nxt_py_content_length_str, &f->value, + RC(nxt_python_add_sptr(pctx, nxt_py_content_length_str, &f->value, f->value_length)); } if (r->content_type_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_type_field; - RC(nxt_python_add_sptr(ctx, nxt_py_content_type_str, &f->value, + RC(nxt_python_add_sptr(pctx, nxt_py_content_type_str, &f->value, f->value_length)); } #undef RC + if (nxt_slow_path(PyDict_SetItem(environ, nxt_py_wsgi_input_str, + (PyObject *) pctx) != 0)) + { + nxt_unit_req_error(pctx->req, + "Python failed to set the \"wsgi.input\" environ value"); + goto fail; + } + return environ; fail: @@ -601,7 +647,7 @@ fail: static int -nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, +nxt_python_add_sptr(nxt_python_ctx_t *pctx, PyObject *name, nxt_unit_sptr_t *sptr, uint32_t size) { char *src; @@ -611,7 +657,7 @@ nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, value = PyString_FromStringAndSize(src, size); if (nxt_slow_path(value == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to create value string \"%.*s\"", (int) size, src); nxt_python_print_exception(); @@ -619,8 +665,8 @@ nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, return NXT_UNIT_ERROR; } - if (nxt_slow_path(PyDict_SetItem(ctx->environ, name, value) != 0)) { - nxt_unit_req_error(ctx->req, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, name, value) != 0)) { + nxt_unit_req_error(pctx->req, "Python failed to set the \"%s\" environ value", PyUnicode_AsUTF8(name)); Py_DECREF(value); @@ -635,7 +681,7 @@ nxt_python_add_sptr(nxt_python_run_ctx_t *ctx, PyObject *name, static int -nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, +nxt_python_add_field(nxt_python_ctx_t *pctx, nxt_unit_field_t *field, int n, uint32_t vl) { char *src; @@ -645,7 +691,7 @@ nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, name = nxt_python_field_name(src, field->name_length); if (nxt_slow_path(name == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to create name string \"%.*s\"", (int) field->name_length, src); nxt_python_print_exception(); @@ -656,7 +702,7 @@ nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, value = nxt_python_field_value(field, n, vl); if (nxt_slow_path(value == NULL)) { - nxt_unit_req_error(ctx->req, + nxt_unit_req_error(pctx->req, "Python failed to create value string \"%.*s\"", (int) field->value_length, (char *) nxt_unit_sptr_get(&field->value)); @@ -665,8 +711,8 @@ nxt_python_add_field(nxt_python_run_ctx_t *ctx, nxt_unit_field_t *field, int n, goto fail; } - if (nxt_slow_path(PyDict_SetItem(ctx->environ, name, value) != 0)) { - nxt_unit_req_error(ctx->req, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, name, value) != 0)) { + nxt_unit_req_error(pctx->req, "Python failed to set the \"%s\" environ value", PyUnicode_AsUTF8(name)); goto fail; @@ -761,10 +807,10 @@ nxt_python_field_value(nxt_unit_field_t *f, int n, uint32_t vl) static int -nxt_python_add_obj(nxt_python_run_ctx_t *ctx, PyObject *name, PyObject *value) +nxt_python_add_obj(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value) { - if (nxt_slow_path(PyDict_SetItem(ctx->environ, name, value) != 0)) { - nxt_unit_req_error(ctx->req, + if (nxt_slow_path(PyDict_SetItem(pctx->environ, name, value) != 0)) { + nxt_unit_req_error(pctx->req, "Python failed to set the \"%s\" environ value", PyUnicode_AsUTF8(name)); @@ -778,15 +824,15 @@ nxt_python_add_obj(nxt_python_run_ctx_t *ctx, PyObject *name, PyObject *value) static PyObject * nxt_py_start_resp(PyObject *self, PyObject *args) { - int rc, status; - char *status_str, *space_ptr; - uint32_t status_len; - PyObject *headers, *tuple, *string, *status_bytes; - Py_ssize_t i, n, fields_size, fields_count; - nxt_python_run_ctx_t *ctx; - - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + int rc, status; + char *status_str, *space_ptr; + uint32_t status_len; + PyObject *headers, *tuple, *string, *status_bytes; + Py_ssize_t i, n, fields_size, fields_count; + nxt_python_ctx_t *pctx; + + pctx = (nxt_python_ctx_t *) self; + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "start_response() is called " "outside of WSGI request processing"); @@ -831,7 +877,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) fields_size += PyBytes_GET_SIZE(string); } else if (PyUnicode_Check(string)) { - fields_size += PyUnicode_GET_SIZE(string); + fields_size += PyUnicode_GET_LENGTH(string); } else { return PyErr_Format(PyExc_TypeError, @@ -843,7 +889,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) fields_size += PyBytes_GET_SIZE(string); } else if (PyUnicode_Check(string)) { - fields_size += PyUnicode_GET_SIZE(string); + fields_size += PyUnicode_GET_LENGTH(string); } else { return PyErr_Format(PyExc_TypeError, @@ -851,7 +897,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) } } - ctx->content_length = -1; + pctx->content_length = -1; string = PyTuple_GET_ITEM(args, 0); rc = nxt_python_str_buf(string, &status_str, &status_len, &status_bytes); @@ -877,7 +923,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) * ... applications can replace their originally intended output with error * output, up until the last possible moment. */ - rc = nxt_unit_response_init(ctx->req, status, fields_count, fields_size); + rc = nxt_unit_response_init(pctx->req, status, fields_count, fields_size); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, "failed to allocate response object"); @@ -886,7 +932,7 @@ nxt_py_start_resp(PyObject *self, PyObject *args) for (i = 0; i < fields_count; i++) { tuple = PyList_GET_ITEM(headers, i); - rc = nxt_python_response_add_field(ctx, PyTuple_GET_ITEM(tuple, 0), + rc = nxt_python_response_add_field(pctx, PyTuple_GET_ITEM(tuple, 0), PyTuple_GET_ITEM(tuple, 1), i); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, @@ -907,21 +953,21 @@ nxt_py_start_resp(PyObject *self, PyObject *args) * possible exception to this rule is if the response headers explicitly * include a Content-Length of zero.) */ - if (ctx->content_length == 0) { - rc = nxt_unit_response_send(ctx->req); + if (pctx->content_length == 0) { + rc = nxt_unit_response_send(pctx->req); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, "failed to send response headers"); } } - Py_INCREF(nxt_py_write_obj); - return nxt_py_write_obj; + Py_INCREF(pctx->write); + return pctx->write; } static int -nxt_python_response_add_field(nxt_python_run_ctx_t *ctx, PyObject *name, +nxt_python_response_add_field(nxt_python_ctx_t *pctx, PyObject *name, PyObject *value, int i) { int rc; @@ -943,20 +989,20 @@ nxt_python_response_add_field(nxt_python_run_ctx_t *ctx, PyObject *name, goto fail; } - rc = nxt_unit_response_add_field(ctx->req, name_str, name_length, + rc = nxt_unit_response_add_field(pctx->req, name_str, name_length, value_str, value_length); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } - if (ctx->req->response->fields[i].hash == NXT_UNIT_HASH_CONTENT_LENGTH) { + if (pctx->req->response->fields[i].hash == NXT_UNIT_HASH_CONTENT_LENGTH) { content_length = nxt_off_t_parse((u_char *) value_str, value_length); if (nxt_slow_path(content_length < 0)) { - nxt_unit_req_error(ctx->req, "failed to parse Content-Length " + nxt_unit_req_error(pctx->req, "failed to parse Content-Length " "value %.*s", (int) value_length, value_str); } else { - ctx->content_length = content_length; + pctx->content_length = content_length; } } @@ -1001,7 +1047,7 @@ nxt_py_write(PyObject *self, PyObject *str) NXT_PYTHON_BYTES_TYPE); } - rc = nxt_python_write(nxt_python_run_ctx, str); + rc = nxt_python_write((nxt_python_ctx_t *) self, str); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return PyErr_Format(PyExc_RuntimeError, "failed to write response value"); @@ -1012,28 +1058,26 @@ nxt_py_write(PyObject *self, PyObject *str) static void -nxt_py_input_dealloc(nxt_py_input_t *self) +nxt_py_input_dealloc(nxt_python_ctx_t *self) { PyObject_Del(self); } static PyObject * -nxt_py_input_read(nxt_py_input_t *self, PyObject *args) +nxt_py_input_read(nxt_python_ctx_t *pctx, PyObject *args) { - char *buf; - PyObject *content, *obj; - Py_ssize_t size, n; - nxt_python_run_ctx_t *ctx; + char *buf; + PyObject *content, *obj; + Py_ssize_t size, n; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.read() is called " "outside of WSGI request processing"); } - size = ctx->req->content_length; + size = pctx->req->content_length; n = PyTuple_GET_SIZE(args); @@ -1057,8 +1101,8 @@ nxt_py_input_read(nxt_py_input_t *self, PyObject *args) } } - if (size == -1 || size > (Py_ssize_t) ctx->req->content_length) { - size = ctx->req->content_length; + if (size == -1 || size > (Py_ssize_t) pctx->req->content_length) { + size = pctx->req->content_length; } } @@ -1069,22 +1113,20 @@ nxt_py_input_read(nxt_py_input_t *self, PyObject *args) buf = PyBytes_AS_STRING(content); - size = nxt_unit_request_read(ctx->req, buf, size); + size = nxt_unit_request_read(pctx->req, buf, size); return content; } static PyObject * -nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) +nxt_py_input_readline(nxt_python_ctx_t *pctx, PyObject *args) { - ssize_t ssize; - PyObject *obj; - Py_ssize_t n; - nxt_python_run_ctx_t *ctx; + ssize_t ssize; + PyObject *obj; + Py_ssize_t n; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.readline() is called " "outside of WSGI request processing"); @@ -1102,7 +1144,7 @@ nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) ssize = PyNumber_AsSsize_t(obj, PyExc_OverflowError); if (nxt_fast_path(ssize > 0)) { - return nxt_py_input_getline(ctx, ssize); + return nxt_py_input_getline(pctx, ssize); } if (ssize == 0) { @@ -1119,18 +1161,18 @@ nxt_py_input_readline(nxt_py_input_t *self, PyObject *args) } } - return nxt_py_input_getline(ctx, SSIZE_MAX); + return nxt_py_input_getline(pctx, SSIZE_MAX); } static PyObject * -nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size) +nxt_py_input_getline(nxt_python_ctx_t *pctx, size_t size) { void *buf; ssize_t res; PyObject *content; - res = nxt_unit_request_readline_size(ctx->req, size); + res = nxt_unit_request_readline_size(pctx->req, size); if (nxt_slow_path(res < 0)) { return NULL; } @@ -1146,20 +1188,18 @@ nxt_py_input_getline(nxt_python_run_ctx_t *ctx, size_t size) buf = PyBytes_AS_STRING(content); - res = nxt_unit_request_read(ctx->req, buf, res); + res = nxt_unit_request_read(pctx->req, buf, res); return content; } static PyObject * -nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args) +nxt_py_input_readlines(nxt_python_ctx_t *pctx, PyObject *args) { - PyObject *res; - nxt_python_run_ctx_t *ctx; + PyObject *res; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.readlines() is called " "outside of WSGI request processing"); @@ -1171,7 +1211,7 @@ nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args) } for ( ;; ) { - PyObject *line = nxt_py_input_getline(ctx, SSIZE_MAX); + PyObject *line = nxt_py_input_getline(pctx, SSIZE_MAX); if (nxt_slow_path(line == NULL)) { Py_DECREF(res); return NULL; @@ -1201,17 +1241,17 @@ nxt_py_input_iter(PyObject *self) static PyObject * nxt_py_input_next(PyObject *self) { - PyObject *line; - nxt_python_run_ctx_t *ctx; + PyObject *line; + nxt_python_ctx_t *pctx; - ctx = nxt_python_run_ctx; - if (nxt_slow_path(ctx == NULL)) { + pctx = (nxt_python_ctx_t *) self; + if (nxt_slow_path(pctx->req == NULL)) { return PyErr_Format(PyExc_RuntimeError, "wsgi.input.next() is called " "outside of WSGI request processing"); } - line = nxt_py_input_getline(ctx, SSIZE_MAX); + line = nxt_py_input_getline(pctx, SSIZE_MAX); if (nxt_slow_path(line == NULL)) { return NULL; } @@ -1227,7 +1267,7 @@ nxt_py_input_next(PyObject *self) static int -nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes) +nxt_python_write(nxt_python_ctx_t *pctx, PyObject *bytes) { int rc; char *str_buf; @@ -1248,16 +1288,16 @@ nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes) * stop iterating over the response when enough data has been sent, or raise * an error if the application tries to write() past that point. */ - if (nxt_slow_path(str_length > ctx->content_length - ctx->bytes_sent)) { - nxt_unit_req_error(ctx->req, "content length %"PRIu64" exceeded", - ctx->content_length); + if (nxt_slow_path(str_length > pctx->content_length - pctx->bytes_sent)) { + nxt_unit_req_error(pctx->req, "content length %"PRIu64" exceeded", + pctx->content_length); return NXT_UNIT_ERROR; } - rc = nxt_unit_response_write(ctx->req, str_buf, str_length); + rc = nxt_unit_response_write(pctx->req, str_buf, str_length); if (nxt_fast_path(rc == NXT_UNIT_OK)) { - ctx->bytes_sent += str_length; + pctx->bytes_sent += str_length; } return rc; diff --git a/src/ruby/nxt_ruby.c b/src/ruby/nxt_ruby.c index 743bf646..698d4a43 100644 --- a/src/ruby/nxt_ruby.c +++ b/src/ruby/nxt_ruby.c @@ -8,32 +8,25 @@ #include <nxt_unit.h> #include <nxt_unit_request.h> +#include <ruby/thread.h> + #include NXT_RUBY_MOUNTS_H #define NXT_RUBY_RACK_API_VERSION_MAJOR 1 #define NXT_RUBY_RACK_API_VERSION_MINOR 3 -#define NXT_RUBY_STRINGIZE_HELPER(x) #x -#define NXT_RUBY_STRINGIZE(x) NXT_RUBY_STRINGIZE_HELPER(x) - -#define NXT_RUBY_LIB_VERSION \ - NXT_RUBY_STRINGIZE(RUBY_API_VERSION_MAJOR) \ - "." NXT_RUBY_STRINGIZE(RUBY_API_VERSION_MINOR) \ - "." NXT_RUBY_STRINGIZE(RUBY_API_VERSION_TEENY) - typedef struct { - nxt_task_t *task; - nxt_str_t *script; - VALUE builder; + nxt_task_t *task; + nxt_str_t *script; + nxt_ruby_ctx_t *rctx; } nxt_ruby_rack_init_t; static nxt_int_t nxt_ruby_start(nxt_task_t *task, nxt_process_data_t *data); static VALUE nxt_ruby_init_basic(VALUE arg); -static nxt_int_t nxt_ruby_init_io(nxt_task_t *task); static VALUE nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init); static VALUE nxt_ruby_require_rubygems(VALUE arg); @@ -41,26 +34,40 @@ static VALUE nxt_ruby_bundler_setup(VALUE arg); static VALUE nxt_ruby_require_rack(VALUE arg); static VALUE nxt_ruby_rack_parse_script(VALUE ctx); static VALUE nxt_ruby_rack_env_create(VALUE arg); +static int nxt_ruby_init_io(nxt_ruby_ctx_t *rctx); static void nxt_ruby_request_handler(nxt_unit_request_info_t *req); +static void *nxt_ruby_request_handler_gvl(void *req); +static int nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx); +static VALUE nxt_ruby_thread_func(VALUE arg); +static void *nxt_ruby_unit_run(void *ctx); +static void nxt_ruby_ubf(void *ctx); +static int nxt_ruby_init_threads(nxt_ruby_app_conf_t *c); +static void nxt_ruby_join_threads(nxt_unit_ctx_t *ctx, + nxt_ruby_app_conf_t *c); static VALUE nxt_ruby_rack_app_run(VALUE arg); -static int nxt_ruby_read_request(VALUE hash_env); -nxt_inline void nxt_ruby_add_sptr(VALUE hash_env, - const char *name, uint32_t name_len, nxt_unit_sptr_t *sptr, uint32_t len); -nxt_inline void nxt_ruby_add_str(VALUE hash_env, - const char *name, uint32_t name_len, const char *str, uint32_t len); -static nxt_int_t nxt_ruby_rack_result_status(VALUE result); -static int nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status); +static int nxt_ruby_read_request(nxt_unit_request_info_t *req, VALUE hash_env); +nxt_inline void nxt_ruby_add_sptr(VALUE hash_env, VALUE name, + nxt_unit_sptr_t *sptr, uint32_t len); +static nxt_int_t nxt_ruby_rack_result_status(nxt_unit_request_info_t *req, + VALUE result); +static int nxt_ruby_rack_result_headers(nxt_unit_request_info_t *req, + VALUE result, nxt_int_t status); static int nxt_ruby_hash_info(VALUE r_key, VALUE r_value, VALUE arg); static int nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg); -static int nxt_ruby_rack_result_body(VALUE result); -static int nxt_ruby_rack_result_body_file_write(VALUE filepath); +static int nxt_ruby_rack_result_body(nxt_unit_request_info_t *req, + VALUE result); +static int nxt_ruby_rack_result_body_file_write(nxt_unit_request_info_t *req, + VALUE filepath); +static void *nxt_ruby_response_write_cb(void *read_info); static VALUE nxt_ruby_rack_result_body_each(VALUE body, VALUE arg, int argc, const VALUE *argv, VALUE blockarg); +static void *nxt_ruby_response_write(void *body); -static void nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, - const char *desc); +static void nxt_ruby_exception_log(nxt_unit_request_info_t *req, + uint32_t level, const char *desc); +static void nxt_ruby_ctx_done(nxt_ruby_ctx_t *rctx); static void nxt_ruby_atexit(void); @@ -68,12 +75,11 @@ static uint32_t compat[] = { NXT_VERNUM, NXT_DEBUG, }; -static VALUE nxt_ruby_rackup; -static VALUE nxt_ruby_call; -static VALUE nxt_ruby_env; -static VALUE nxt_ruby_io_input; -static VALUE nxt_ruby_io_error; -static nxt_ruby_run_ctx_t nxt_ruby_run_ctx; +static VALUE nxt_ruby_rackup; +static VALUE nxt_ruby_call; + +static uint32_t nxt_ruby_threads; +static nxt_ruby_ctx_t *nxt_ruby_ctxs; NXT_EXPORT nxt_app_module_t nxt_app_module = { sizeof(compat), @@ -86,83 +92,207 @@ NXT_EXPORT nxt_app_module_t nxt_app_module = { nxt_ruby_start, }; +typedef struct { + nxt_str_t string; + VALUE *v; +} nxt_ruby_string_t; + +static VALUE nxt_rb_80_str; +static VALUE nxt_rb_content_length_str; +static VALUE nxt_rb_content_type_str; +static VALUE nxt_rb_http_str; +static VALUE nxt_rb_https_str; +static VALUE nxt_rb_path_info_str; +static VALUE nxt_rb_query_string_str; +static VALUE nxt_rb_rack_url_scheme_str; +static VALUE nxt_rb_remote_addr_str; +static VALUE nxt_rb_request_method_str; +static VALUE nxt_rb_request_uri_str; +static VALUE nxt_rb_server_addr_str; +static VALUE nxt_rb_server_name_str; +static VALUE nxt_rb_server_port_str; +static VALUE nxt_rb_server_protocol_str; + +static nxt_ruby_string_t nxt_rb_strings[] = { + { nxt_string("80"), &nxt_rb_80_str }, + { nxt_string("CONTENT_LENGTH"), &nxt_rb_content_length_str }, + { nxt_string("CONTENT_TYPE"), &nxt_rb_content_type_str }, + { nxt_string("http"), &nxt_rb_http_str }, + { nxt_string("https"), &nxt_rb_https_str }, + { nxt_string("PATH_INFO"), &nxt_rb_path_info_str }, + { nxt_string("QUERY_STRING"), &nxt_rb_query_string_str }, + { nxt_string("rack.url_scheme"), &nxt_rb_rack_url_scheme_str }, + { nxt_string("REMOTE_ADDR"), &nxt_rb_remote_addr_str }, + { nxt_string("REQUEST_METHOD"), &nxt_rb_request_method_str }, + { nxt_string("REQUEST_URI"), &nxt_rb_request_uri_str }, + { nxt_string("SERVER_ADDR"), &nxt_rb_server_addr_str }, + { nxt_string("SERVER_NAME"), &nxt_rb_server_name_str }, + { nxt_string("SERVER_PORT"), &nxt_rb_server_port_str }, + { nxt_string("SERVER_PROTOCOL"), &nxt_rb_server_protocol_str }, + { nxt_null_string, NULL }, +}; + + +static int +nxt_ruby_init_strings(void) +{ + VALUE v; + nxt_ruby_string_t *pstr; + + pstr = nxt_rb_strings; + + while (pstr->string.start != NULL) { + v = rb_str_new_static((char *) pstr->string.start, pstr->string.length); + + if (nxt_slow_path(v == Qnil)) { + nxt_unit_alert(NULL, "Ruby: failed to create const string '%.*s'", + (int) pstr->string.length, + (char *) pstr->string.start); + + return NXT_UNIT_ERROR; + } + + *pstr->v = v; + + rb_gc_register_address(pstr->v); + + pstr++; + } + + return NXT_UNIT_OK; +} + + +static void +nxt_ruby_done_strings(void) +{ + nxt_ruby_string_t *pstr; + + pstr = nxt_rb_strings; + + while (pstr->string.start != NULL) { + rb_gc_unregister_address(pstr->v); + + *pstr->v = Qnil; + + pstr++; + } +} + static nxt_int_t nxt_ruby_start(nxt_task_t *task, nxt_process_data_t *data) { int state, rc; VALUE res; + nxt_ruby_ctx_t ruby_ctx; nxt_unit_ctx_t *unit_ctx; nxt_unit_init_t ruby_unit_init; + nxt_ruby_app_conf_t *c; nxt_ruby_rack_init_t rack_init; nxt_common_app_conf_t *conf; static char *argv[2] = { (char *) "NGINX_Unit", (char *) "-e0" }; conf = data->app; + c = &conf->u.ruby; + + nxt_ruby_threads = c->threads; RUBY_INIT_STACK ruby_init(); ruby_options(2, argv); ruby_script("NGINX_Unit"); + ruby_ctx.env = Qnil; + ruby_ctx.io_input = Qnil; + ruby_ctx.io_error = Qnil; + ruby_ctx.thread = Qnil; + ruby_ctx.ctx = NULL; + ruby_ctx.req = NULL; + rack_init.task = task; - rack_init.script = &conf->u.ruby.script; + rack_init.script = &c->script; + rack_init.rctx = &ruby_ctx; + + nxt_ruby_init_strings(); res = rb_protect(nxt_ruby_init_basic, (VALUE) (uintptr_t) &rack_init, &state); if (nxt_slow_path(res == Qnil || state != 0)) { - nxt_ruby_exception_log(task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to init basic variables"); return NXT_ERROR; } + nxt_ruby_call = Qnil; + nxt_ruby_rackup = nxt_ruby_rack_init(&rack_init); if (nxt_slow_path(nxt_ruby_rackup == Qnil)) { return NXT_ERROR; } + rb_gc_register_address(&nxt_ruby_rackup); + nxt_ruby_call = rb_intern("call"); if (nxt_slow_path(nxt_ruby_call == Qnil)) { nxt_alert(task, "Ruby: Unable to find rack entry point"); - return NXT_ERROR; + goto fail; } - nxt_ruby_env = rb_protect(nxt_ruby_rack_env_create, Qnil, &state); - if (nxt_slow_path(state != 0)) { - nxt_ruby_exception_log(task, NXT_LOG_ALERT, + rb_gc_register_address(&nxt_ruby_call); + + ruby_ctx.env = rb_protect(nxt_ruby_rack_env_create, + (VALUE) (uintptr_t) &ruby_ctx, &state); + if (nxt_slow_path(ruby_ctx.env == Qnil || state != 0)) { + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to create 'environ' variable"); - return NXT_ERROR; + goto fail; } - rb_gc_register_address(&nxt_ruby_rackup); - rb_gc_register_address(&nxt_ruby_call); - rb_gc_register_address(&nxt_ruby_env); + rc = nxt_ruby_init_threads(c); + if (nxt_slow_path(rc == NXT_UNIT_ERROR)) { + goto fail; + } nxt_unit_default_init(task, &ruby_unit_init); ruby_unit_init.callbacks.request_handler = nxt_ruby_request_handler; + ruby_unit_init.callbacks.ready_handler = nxt_ruby_ready_handler; ruby_unit_init.shm_limit = conf->shm_limit; + ruby_unit_init.data = c; + ruby_unit_init.ctx_data = &ruby_ctx; unit_ctx = nxt_unit_init(&ruby_unit_init); if (nxt_slow_path(unit_ctx == NULL)) { - return NXT_ERROR; + goto fail; } - nxt_ruby_run_ctx.unit_ctx = unit_ctx; + rc = (intptr_t) rb_thread_call_without_gvl(nxt_ruby_unit_run, unit_ctx, + nxt_ruby_ubf, unit_ctx); - rc = nxt_unit_run(unit_ctx); + nxt_ruby_join_threads(unit_ctx, c); - nxt_ruby_atexit(); + nxt_unit_done(unit_ctx); - nxt_ruby_run_ctx.unit_ctx = NULL; + nxt_ruby_ctx_done(&ruby_ctx); - nxt_unit_done(unit_ctx); + nxt_ruby_atexit(); exit(rc); return NXT_OK; + +fail: + + nxt_ruby_join_threads(NULL, c); + + nxt_ruby_ctx_done(&ruby_ctx); + + nxt_ruby_atexit(); + + return NXT_ERROR; } @@ -170,7 +300,6 @@ static VALUE nxt_ruby_init_basic(VALUE arg) { int state; - nxt_int_t rc; nxt_ruby_rack_init_t *rack_init; rack_init = (nxt_ruby_rack_init_t *) (uintptr_t) arg; @@ -186,56 +315,19 @@ nxt_ruby_init_basic(VALUE arg) rb_funcall(rb_cObject, rb_intern("require"), 1, rb_str_new2("enc/trans/transdb")); - rc = nxt_ruby_init_io(rack_init->task); - if (nxt_slow_path(rc != NXT_OK)) { - return Qnil; - } - return arg; } -static nxt_int_t -nxt_ruby_init_io(nxt_task_t *task) -{ - VALUE rb, io_input, io_error; - - io_input = nxt_ruby_stream_io_input_init(); - rb = Data_Wrap_Struct(io_input, 0, 0, &nxt_ruby_run_ctx); - - nxt_ruby_io_input = rb_funcall(io_input, rb_intern("new"), 1, rb); - if (nxt_slow_path(nxt_ruby_io_input == Qnil)) { - nxt_alert(task, "Ruby: Failed to create environment 'rack.input' var"); - - return NXT_ERROR; - } - - io_error = nxt_ruby_stream_io_error_init(); - rb = Data_Wrap_Struct(io_error, 0, 0, &nxt_ruby_run_ctx); - - nxt_ruby_io_error = rb_funcall(io_error, rb_intern("new"), 1, rb); - if (nxt_slow_path(nxt_ruby_io_error == Qnil)) { - nxt_alert(task, "Ruby: Failed to create environment 'rack.error' var"); - - return NXT_ERROR; - } - - rb_gc_register_address(&nxt_ruby_io_input); - rb_gc_register_address(&nxt_ruby_io_error); - - return NXT_OK; -} - - static VALUE nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init) { int state; - VALUE rack, rackup, err; + VALUE rackup, err; rb_protect(nxt_ruby_require_rubygems, Qnil, &state); if (nxt_slow_path(state != 0)) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to require 'rubygems' package"); return Qnil; } @@ -245,7 +337,7 @@ nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init) err = rb_errinfo(); if (rb_obj_is_kind_of(err, rb_eLoadError) == Qfalse) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to require 'bundler/setup' package"); return Qnil; } @@ -255,18 +347,15 @@ nxt_ruby_rack_init(nxt_ruby_rack_init_t *rack_init) rb_protect(nxt_ruby_require_rack, Qnil, &state); if (nxt_slow_path(state != 0)) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to require 'rack' package"); return Qnil; } - rack = rb_const_get(rb_cObject, rb_intern("Rack")); - rack_init->builder = rb_const_get(rack, rb_intern("Builder")); - rackup = rb_protect(nxt_ruby_rack_parse_script, (VALUE) (uintptr_t) rack_init, &state); if (nxt_slow_path(TYPE(rackup) != T_ARRAY || state != 0)) { - nxt_ruby_exception_log(rack_init->task, NXT_LOG_ALERT, + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, "Failed to parse rack script"); return Qnil; } @@ -306,15 +395,18 @@ nxt_ruby_require_rack(VALUE arg) static VALUE nxt_ruby_rack_parse_script(VALUE ctx) { - VALUE script, res; + VALUE script, res, rack, builder; nxt_ruby_rack_init_t *rack_init; rack_init = (nxt_ruby_rack_init_t *) (uintptr_t) ctx; + rack = rb_const_get(rb_cObject, rb_intern("Rack")); + builder = rb_const_get(rack, rb_intern("Builder")); + script = rb_str_new((const char *) rack_init->script->start, (long) rack_init->script->length); - res = rb_funcall(rack_init->builder, rb_intern("parse_file"), 1, script); + res = rb_funcall(builder, rb_intern("parse_file"), 1, script); rb_str_free(script); @@ -325,7 +417,16 @@ nxt_ruby_rack_parse_script(VALUE ctx) static VALUE nxt_ruby_rack_env_create(VALUE arg) { - VALUE hash_env, version; + int rc; + VALUE hash_env, version; + nxt_ruby_ctx_t *rctx; + + rctx = (nxt_ruby_ctx_t *) (uintptr_t) arg; + + rc = nxt_ruby_init_io(rctx); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + return Qnil; + } hash_env = rb_hash_new(); @@ -339,47 +440,114 @@ nxt_ruby_rack_env_create(VALUE arg) rb_ary_push(version, UINT2NUM(NXT_RUBY_RACK_API_VERSION_MINOR)); rb_hash_aset(hash_env, rb_str_new2("rack.version"), version); - rb_hash_aset(hash_env, rb_str_new2("rack.input"), nxt_ruby_io_input); - rb_hash_aset(hash_env, rb_str_new2("rack.errors"), nxt_ruby_io_error); - rb_hash_aset(hash_env, rb_str_new2("rack.multithread"), Qfalse); + rb_hash_aset(hash_env, rb_str_new2("rack.input"), rctx->io_input); + rb_hash_aset(hash_env, rb_str_new2("rack.errors"), rctx->io_error); + rb_hash_aset(hash_env, rb_str_new2("rack.multithread"), + nxt_ruby_threads > 1 ? Qtrue : Qfalse); rb_hash_aset(hash_env, rb_str_new2("rack.multiprocess"), Qtrue); rb_hash_aset(hash_env, rb_str_new2("rack.run_once"), Qfalse); rb_hash_aset(hash_env, rb_str_new2("rack.hijack?"), Qfalse); rb_hash_aset(hash_env, rb_str_new2("rack.hijack"), Qnil); rb_hash_aset(hash_env, rb_str_new2("rack.hijack_io"), Qnil); + rctx->env = hash_env; + + rb_gc_register_address(&rctx->env); + return hash_env; } +static int +nxt_ruby_init_io(nxt_ruby_ctx_t *rctx) +{ + VALUE io_input, io_error; + + io_input = nxt_ruby_stream_io_input_init(); + + rctx->io_input = rb_funcall(io_input, rb_intern("new"), 1, + (VALUE) (uintptr_t) rctx); + if (nxt_slow_path(rctx->io_input == Qnil)) { + nxt_unit_alert(NULL, + "Ruby: Failed to create environment 'rack.input' var"); + + return NXT_UNIT_ERROR; + } + + rb_gc_register_address(&rctx->io_input); + + io_error = nxt_ruby_stream_io_error_init(); + + rctx->io_error = rb_funcall(io_error, rb_intern("new"), 1, + (VALUE) (uintptr_t) rctx); + if (nxt_slow_path(rctx->io_error == Qnil)) { + nxt_unit_alert(NULL, + "Ruby: Failed to create environment 'rack.error' var"); + + return NXT_UNIT_ERROR; + } + + rb_gc_register_address(&rctx->io_error); + + return NXT_UNIT_OK; +} + + static void nxt_ruby_request_handler(nxt_unit_request_info_t *req) { - int state; - VALUE res; + (void) rb_thread_call_with_gvl(nxt_ruby_request_handler_gvl, req); +} - nxt_ruby_run_ctx.req = req; - res = rb_protect(nxt_ruby_rack_app_run, Qnil, &state); +static void * +nxt_ruby_request_handler_gvl(void *data) +{ + int state; + VALUE res; + nxt_ruby_ctx_t *rctx; + nxt_unit_request_info_t *req; + + req = data; + + rctx = req->ctx->data; + rctx->req = req; + + res = rb_protect(nxt_ruby_rack_app_run, (VALUE) (uintptr_t) req, &state); if (nxt_slow_path(res == Qnil || state != 0)) { - nxt_ruby_exception_log(NULL, NXT_LOG_ERR, + nxt_ruby_exception_log(req, NXT_LOG_ERR, "Failed to run ruby script"); + + nxt_unit_request_done(req, NXT_UNIT_ERROR); + + } else { + nxt_unit_request_done(req, NXT_UNIT_OK); } + + rctx->req = NULL; + + return NULL; } static VALUE nxt_ruby_rack_app_run(VALUE arg) { - int rc; - VALUE env, result; - nxt_int_t status; + int rc; + VALUE env, result; + nxt_int_t status; + nxt_ruby_ctx_t *rctx; + nxt_unit_request_info_t *req; - env = rb_hash_dup(nxt_ruby_env); + req = (nxt_unit_request_info_t *) arg; - rc = nxt_ruby_read_request(env); + rctx = req->ctx->data; + + env = rb_hash_dup(rctx->env); + + rc = nxt_ruby_read_request(req, env); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_req_alert(nxt_ruby_run_ctx.req, + nxt_unit_req_alert(req, "Ruby: Failed to process incoming request"); goto fail; @@ -387,50 +555,44 @@ nxt_ruby_rack_app_run(VALUE arg) result = rb_funcall(nxt_ruby_rackup, nxt_ruby_call, 1, env); if (nxt_slow_path(TYPE(result) != T_ARRAY)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response format from application"); goto fail; } if (nxt_slow_path(RARRAY_LEN(result) != 3)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response format from application. " "Need 3 entries [Status, Headers, Body]"); goto fail; } - status = nxt_ruby_rack_result_status(result); + status = nxt_ruby_rack_result_status(req, result); if (nxt_slow_path(status < 0)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response status from application."); goto fail; } - rc = nxt_ruby_rack_result_headers(result, status); + rc = nxt_ruby_rack_result_headers(req, result, status); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } - rc = nxt_ruby_rack_result_body(result); + rc = nxt_ruby_rack_result_body(req, result); if (nxt_slow_path(rc != NXT_UNIT_OK)) { goto fail; } - nxt_unit_request_done(nxt_ruby_run_ctx.req, rc); - nxt_ruby_run_ctx.req = NULL; - rb_hash_delete(env, rb_obj_id(env)); return result; fail: - nxt_unit_request_done(nxt_ruby_run_ctx.req, NXT_UNIT_ERROR); - nxt_ruby_run_ctx.req = NULL; - rb_hash_delete(env, rb_obj_id(env)); return Qnil; @@ -438,85 +600,76 @@ fail: static int -nxt_ruby_read_request(VALUE hash_env) +nxt_ruby_read_request(nxt_unit_request_info_t *req, VALUE hash_env) { + VALUE name; uint32_t i; nxt_unit_field_t *f; nxt_unit_request_t *r; - r = nxt_ruby_run_ctx.req->request; + r = req->request; -#define NL(S) (S), sizeof(S)-1 - - nxt_ruby_add_sptr(hash_env, NL("REQUEST_METHOD"), &r->method, + nxt_ruby_add_sptr(hash_env, nxt_rb_request_method_str, &r->method, r->method_length); - nxt_ruby_add_sptr(hash_env, NL("REQUEST_URI"), &r->target, + nxt_ruby_add_sptr(hash_env, nxt_rb_request_uri_str, &r->target, r->target_length); - nxt_ruby_add_sptr(hash_env, NL("PATH_INFO"), &r->path, r->path_length); - nxt_ruby_add_sptr(hash_env, NL("QUERY_STRING"), &r->query, + nxt_ruby_add_sptr(hash_env, nxt_rb_path_info_str, &r->path, r->path_length); + nxt_ruby_add_sptr(hash_env, nxt_rb_query_string_str, &r->query, r->query_length); - nxt_ruby_add_sptr(hash_env, NL("SERVER_PROTOCOL"), &r->version, + nxt_ruby_add_sptr(hash_env, nxt_rb_server_protocol_str, &r->version, r->version_length); - nxt_ruby_add_sptr(hash_env, NL("REMOTE_ADDR"), &r->remote, + nxt_ruby_add_sptr(hash_env, nxt_rb_remote_addr_str, &r->remote, r->remote_length); - nxt_ruby_add_sptr(hash_env, NL("SERVER_ADDR"), &r->local, r->local_length); - - nxt_ruby_add_sptr(hash_env, NL("SERVER_NAME"), &r->server_name, + nxt_ruby_add_sptr(hash_env, nxt_rb_server_addr_str, &r->local, + r->local_length); + nxt_ruby_add_sptr(hash_env, nxt_rb_server_name_str, &r->server_name, r->server_name_length); - nxt_ruby_add_str(hash_env, NL("SERVER_PORT"), "80", 2); - rb_hash_aset(hash_env, rb_str_new2("rack.url_scheme"), - r->tls ? rb_str_new2("https") : rb_str_new2("http")); + rb_hash_aset(hash_env, nxt_rb_server_port_str, nxt_rb_80_str); + + rb_hash_aset(hash_env, nxt_rb_rack_url_scheme_str, + r->tls ? nxt_rb_https_str : nxt_rb_http_str); for (i = 0; i < r->fields_count; i++) { f = r->fields + i; - nxt_ruby_add_sptr(hash_env, nxt_unit_sptr_get(&f->name), f->name_length, - &f->value, f->value_length); + name = rb_str_new(nxt_unit_sptr_get(&f->name), f->name_length); + + nxt_ruby_add_sptr(hash_env, name, &f->value, f->value_length); } if (r->content_length_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_length_field; - nxt_ruby_add_sptr(hash_env, NL("CONTENT_LENGTH"), + nxt_ruby_add_sptr(hash_env, nxt_rb_content_length_str, &f->value, f->value_length); } if (r->content_type_field != NXT_UNIT_NONE_FIELD) { f = r->fields + r->content_type_field; - nxt_ruby_add_sptr(hash_env, NL("CONTENT_TYPE"), + nxt_ruby_add_sptr(hash_env, nxt_rb_content_type_str, &f->value, f->value_length); } -#undef NL - return NXT_UNIT_OK; } nxt_inline void -nxt_ruby_add_sptr(VALUE hash_env, - const char *name, uint32_t name_len, nxt_unit_sptr_t *sptr, uint32_t len) +nxt_ruby_add_sptr(VALUE hash_env, VALUE name, + nxt_unit_sptr_t *sptr, uint32_t len) { char *str; str = nxt_unit_sptr_get(sptr); - rb_hash_aset(hash_env, rb_str_new(name, name_len), rb_str_new(str, len)); -} - - -nxt_inline void -nxt_ruby_add_str(VALUE hash_env, - const char *name, uint32_t name_len, const char *str, uint32_t len) -{ - rb_hash_aset(hash_env, rb_str_new(name, name_len), rb_str_new(str, len)); + rb_hash_aset(hash_env, name, rb_str_new(str, len)); } static nxt_int_t -nxt_ruby_rack_result_status(VALUE result) +nxt_ruby_rack_result_status(nxt_unit_request_info_t *req, VALUE result) { VALUE status; @@ -531,7 +684,7 @@ nxt_ruby_rack_result_status(VALUE result) RSTRING_LEN(status)); } - nxt_unit_req_error(nxt_ruby_run_ctx.req, "Ruby: Invalid response 'status' " + nxt_unit_req_error(req, "Ruby: Invalid response 'status' " "format from application"); return -2; @@ -539,14 +692,16 @@ nxt_ruby_rack_result_status(VALUE result) typedef struct { - int rc; - uint32_t fields; - uint32_t size; + int rc; + uint32_t fields; + uint32_t size; + nxt_unit_request_info_t *req; } nxt_ruby_headers_info_t; static int -nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) +nxt_ruby_rack_result_headers(nxt_unit_request_info_t *req, VALUE result, + nxt_int_t status) { int rc; VALUE headers; @@ -554,7 +709,7 @@ nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) headers = rb_ary_entry(result, 1); if (nxt_slow_path(TYPE(headers) != T_HASH)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response 'headers' format from " "application"); @@ -566,6 +721,7 @@ nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) headers_info.rc = NXT_UNIT_OK; headers_info.fields = 0; headers_info.size = 0; + headers_info.req = req; rb_hash_foreach(headers, nxt_ruby_hash_info, (VALUE) (uintptr_t) &headers_info); @@ -573,13 +729,14 @@ nxt_ruby_rack_result_headers(VALUE result, nxt_int_t status) return headers_info.rc; } - rc = nxt_unit_response_init(nxt_ruby_run_ctx.req, status, + rc = nxt_unit_response_init(req, status, headers_info.fields, headers_info.size); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return rc; } - rb_hash_foreach(headers, nxt_ruby_hash_add, (VALUE) (uintptr_t) &rc); + rb_hash_foreach(headers, nxt_ruby_hash_add, + (VALUE) (uintptr_t) &headers_info); return rc; } @@ -594,14 +751,14 @@ nxt_ruby_hash_info(VALUE r_key, VALUE r_value, VALUE arg) headers_info = (void *) (uintptr_t) arg; if (nxt_slow_path(TYPE(r_key) != T_STRING)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(headers_info->req, "Ruby: Wrong header entry 'key' from application"); goto fail; } if (nxt_slow_path(TYPE(r_value) != T_STRING)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(headers_info->req, "Ruby: Wrong header entry 'value' from application"); goto fail; @@ -644,11 +801,13 @@ fail: static int nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg) { - int *rc; - uint32_t key_len; - const char *value, *value_end, *pos; + int *rc; + uint32_t key_len; + const char *value, *value_end, *pos; + nxt_ruby_headers_info_t *headers_info; - rc = (int *) (uintptr_t) arg; + headers_info = (void *) (uintptr_t) arg; + rc = &headers_info->rc; value = RSTRING_PTR(r_value); value_end = value + RSTRING_LEN(r_value); @@ -664,7 +823,7 @@ nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg) break; } - *rc = nxt_unit_response_add_field(nxt_ruby_run_ctx.req, + *rc = nxt_unit_response_add_field(headers_info->req, RSTRING_PTR(r_key), key_len, value, pos - value); if (nxt_slow_path(*rc != NXT_UNIT_OK)) { @@ -676,7 +835,7 @@ nxt_ruby_hash_add(VALUE r_key, VALUE r_value, VALUE arg) } if (value <= value_end) { - *rc = nxt_unit_response_add_field(nxt_ruby_run_ctx.req, + *rc = nxt_unit_response_add_field(headers_info->req, RSTRING_PTR(r_key), key_len, value, value_end - value); if (nxt_slow_path(*rc != NXT_UNIT_OK)) { @@ -695,7 +854,7 @@ fail: static int -nxt_ruby_rack_result_body(VALUE result) +nxt_ruby_rack_result_body(nxt_unit_request_info_t *req, VALUE result) { int rc; VALUE fn, body; @@ -706,24 +865,24 @@ nxt_ruby_rack_result_body(VALUE result) fn = rb_funcall(body, rb_intern("to_path"), 0); if (nxt_slow_path(TYPE(fn) != T_STRING)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Failed to get 'body' file path from " "application"); return NXT_UNIT_ERROR; } - rc = nxt_ruby_rack_result_body_file_write(fn); + rc = nxt_ruby_rack_result_body_file_write(req, fn); if (nxt_slow_path(rc != NXT_UNIT_OK)) { return rc; } } else if (rb_respond_to(body, rb_intern("each"))) { rb_block_call(body, rb_intern("each"), 0, 0, - nxt_ruby_rack_result_body_each, 0); + nxt_ruby_rack_result_body_each, (VALUE) (uintptr_t) req); } else { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Invalid response 'body' format " "from application"); @@ -772,17 +931,24 @@ nxt_ruby_rack_file_read(nxt_unit_read_info_t *read_info, void *dst, size_t size) } +typedef struct { + nxt_unit_read_info_t read_info; + nxt_unit_request_info_t *req; +} nxt_ruby_read_info_t; + + static int -nxt_ruby_rack_result_body_file_write(VALUE filepath) +nxt_ruby_rack_result_body_file_write(nxt_unit_request_info_t *req, + VALUE filepath) { int fd, rc; struct stat finfo; nxt_ruby_rack_file_t ruby_file; - nxt_unit_read_info_t read_info; + nxt_ruby_read_info_t ri; fd = open(RSTRING_PTR(filepath), O_RDONLY, 0); if (nxt_slow_path(fd == -1)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Failed to open content file \"%s\": %s (%d)", RSTRING_PTR(filepath), strerror(errno), errno); @@ -791,7 +957,7 @@ nxt_ruby_rack_result_body_file_write(VALUE filepath) rc = fstat(fd, &finfo); if (nxt_slow_path(rc == -1)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(req, "Ruby: Content file fstat(\"%s\") failed: %s (%d)", RSTRING_PTR(filepath), strerror(errno), errno); @@ -804,16 +970,16 @@ nxt_ruby_rack_result_body_file_write(VALUE filepath) ruby_file.pos = 0; ruby_file.rest = finfo.st_size; - read_info.read = nxt_ruby_rack_file_read; - read_info.eof = ruby_file.rest == 0; - read_info.buf_size = ruby_file.rest; - read_info.data = &ruby_file; + ri.read_info.read = nxt_ruby_rack_file_read; + ri.read_info.eof = ruby_file.rest == 0; + ri.read_info.buf_size = ruby_file.rest; + ri.read_info.data = &ruby_file; + ri.req = req; - rc = nxt_unit_response_write_cb(nxt_ruby_run_ctx.req, &read_info); - if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, - "Ruby: Failed to write content file."); - } + rc = (intptr_t) rb_thread_call_without_gvl(nxt_ruby_response_write_cb, + &ri, + nxt_ruby_ubf, + req->ctx); close(fd); @@ -821,39 +987,77 @@ nxt_ruby_rack_result_body_file_write(VALUE filepath) } +static void * +nxt_ruby_response_write_cb(void *data) +{ + int rc; + nxt_ruby_read_info_t *ri; + + ri = data; + + rc = nxt_unit_response_write_cb(ri->req, &ri->read_info); + if (nxt_slow_path(rc != NXT_UNIT_OK)) { + nxt_unit_req_error(ri->req, "Ruby: Failed to write content file."); + } + + return (void *) (intptr_t) rc; +} + + +typedef struct { + VALUE body; + nxt_unit_request_info_t *req; +} nxt_ruby_write_info_t; + + static VALUE nxt_ruby_rack_result_body_each(VALUE body, VALUE arg, int argc, const VALUE *argv, VALUE blockarg) { - int rc; + nxt_ruby_write_info_t wi; if (TYPE(body) != T_STRING) { return Qnil; } - rc = nxt_unit_response_write(nxt_ruby_run_ctx.req, RSTRING_PTR(body), - RSTRING_LEN(body)); + wi.body = body; + wi.req = (void *) (uintptr_t) arg; + + (void) rb_thread_call_without_gvl(nxt_ruby_response_write, + (void *) (uintptr_t) &wi, + nxt_ruby_ubf, wi.req->ctx); + + return Qnil; +} + + +static void * +nxt_ruby_response_write(void *data) +{ + int rc; + nxt_ruby_write_info_t *wi; + + wi = data; + + rc = nxt_unit_response_write(wi->req, RSTRING_PTR(wi->body), + RSTRING_LEN(wi->body)); if (nxt_slow_path(rc != NXT_UNIT_OK)) { - nxt_unit_req_error(nxt_ruby_run_ctx.req, + nxt_unit_req_error(wi->req, "Ruby: Failed to write 'body' from application"); } - return Qnil; + return (void *) (intptr_t) rc; } static void -nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, const char *desc) +nxt_ruby_exception_log(nxt_unit_request_info_t *req, uint32_t level, + const char *desc) { int i; VALUE err, ary, eclass, msg; - if (task != NULL) { - nxt_log(task, level, "Ruby: %s", desc); - - } else { - nxt_unit_log(nxt_ruby_run_ctx.unit_ctx, level, "Ruby: %s", desc); - } + nxt_unit_req_log(req, level, "Ruby: %s", desc); err = rb_errinfo(); if (nxt_slow_path(err == Qnil)) { @@ -868,25 +1072,30 @@ nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, const char *desc) eclass = rb_class_name(rb_class_of(err)); msg = rb_funcall(err, rb_intern("message"), 0); - if (task != NULL) { - nxt_log(task, level, "Ruby: %s: %s (%s)", - RSTRING_PTR(RARRAY_PTR(ary)[0]), - RSTRING_PTR(msg), RSTRING_PTR(eclass)); - - } else { - nxt_unit_log(nxt_ruby_run_ctx.unit_ctx, level, "Ruby: %s: %s (%s)", + nxt_unit_req_log(req, level, "Ruby: %s: %s (%s)", RSTRING_PTR(RARRAY_PTR(ary)[0]), RSTRING_PTR(msg), RSTRING_PTR(eclass)); - } for (i = 1; i < RARRAY_LEN(ary); i++) { - if (task != NULL) { - nxt_log(task, level, "from %s", RSTRING_PTR(RARRAY_PTR(ary)[i])); - - } else { - nxt_unit_log(nxt_ruby_run_ctx.unit_ctx, level, "from %s", + nxt_unit_req_log(req, level, "from %s", RSTRING_PTR(RARRAY_PTR(ary)[i])); - } + } +} + + +static void +nxt_ruby_ctx_done(nxt_ruby_ctx_t *rctx) +{ + if (rctx->io_input != Qnil) { + rb_gc_unregister_address(&rctx->io_input); + } + + if (rctx->io_error != Qnil) { + rb_gc_unregister_address(&rctx->io_error); + } + + if (rctx->env != Qnil) { + rb_gc_unregister_address(&rctx->env); } } @@ -894,12 +1103,174 @@ nxt_ruby_exception_log(nxt_task_t *task, uint32_t level, const char *desc) static void nxt_ruby_atexit(void) { - rb_gc_unregister_address(&nxt_ruby_io_input); - rb_gc_unregister_address(&nxt_ruby_io_error); + if (nxt_ruby_rackup != Qnil) { + rb_gc_unregister_address(&nxt_ruby_rackup); + } - rb_gc_unregister_address(&nxt_ruby_rackup); - rb_gc_unregister_address(&nxt_ruby_call); - rb_gc_unregister_address(&nxt_ruby_env); + if (nxt_ruby_call != Qnil) { + rb_gc_unregister_address(&nxt_ruby_call); + } + + nxt_ruby_done_strings(); ruby_cleanup(0); } + + +static int +nxt_ruby_ready_handler(nxt_unit_ctx_t *ctx) +{ + VALUE res; + uint32_t i; + nxt_ruby_ctx_t *rctx; + nxt_ruby_app_conf_t *c; + + /* Worker thread context. */ + if (!nxt_unit_is_main_ctx(ctx)) { + return NXT_UNIT_OK; + } + + c = ctx->unit->data; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + rctx->ctx = ctx; + + res = rb_thread_create(RUBY_METHOD_FUNC(nxt_ruby_thread_func), rctx); + + if (nxt_fast_path(res != Qnil)) { + nxt_unit_debug(ctx, "thread #%d created", (int) (i + 1)); + + rctx->thread = res; + + } else { + nxt_unit_alert(ctx, "thread #%d create failed", (int) (i + 1)); + + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static VALUE +nxt_ruby_thread_func(VALUE arg) +{ + nxt_unit_ctx_t *ctx; + nxt_ruby_ctx_t *rctx; + + rctx = (nxt_ruby_ctx_t *) (uintptr_t) arg; + + nxt_unit_debug(rctx->ctx, "worker thread start"); + + ctx = nxt_unit_ctx_alloc(rctx->ctx, rctx); + if (nxt_slow_path(ctx == NULL)) { + goto fail; + } + + (void) rb_thread_call_without_gvl(nxt_ruby_unit_run, ctx, + nxt_ruby_ubf, ctx); + + nxt_unit_done(ctx); + +fail: + + nxt_unit_debug(NULL, "worker thread end"); + + return Qnil; +} + + +static void * +nxt_ruby_unit_run(void *ctx) +{ + return (void *) (intptr_t) nxt_unit_run(ctx); +} + + +static void +nxt_ruby_ubf(void *ctx) +{ + nxt_unit_warn(ctx, "Ruby: UBF"); +} + + +static int +nxt_ruby_init_threads(nxt_ruby_app_conf_t *c) +{ + int state; + uint32_t i; + nxt_ruby_ctx_t *rctx; + + if (c->threads <= 1) { + return NXT_UNIT_OK; + } + + nxt_ruby_ctxs = nxt_unit_malloc(NULL, sizeof(nxt_ruby_ctx_t) + * (c->threads - 1)); + if (nxt_slow_path(nxt_ruby_ctxs == NULL)) { + nxt_unit_alert(NULL, "Failed to allocate run contexts array"); + + return NXT_UNIT_ERROR; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + rctx->env = Qnil; + rctx->io_input = Qnil; + rctx->io_error = Qnil; + rctx->thread = Qnil; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + rctx->env = rb_protect(nxt_ruby_rack_env_create, + (VALUE) (uintptr_t) rctx, &state); + if (nxt_slow_path(rctx->env == Qnil || state != 0)) { + nxt_ruby_exception_log(NULL, NXT_LOG_ALERT, + "Failed to create 'environ' variable"); + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void +nxt_ruby_join_threads(nxt_unit_ctx_t *ctx, nxt_ruby_app_conf_t *c) +{ + uint32_t i; + nxt_ruby_ctx_t *rctx; + + if (nxt_ruby_ctxs == NULL) { + return; + } + + for (i = 0; i < c->threads - 1; i++) { + rctx = &nxt_ruby_ctxs[i]; + + if (rctx->thread != Qnil) { + rb_funcall(rctx->thread, rb_intern("join"), 0); + + nxt_unit_debug(ctx, "thread #%d joined", (int) (i + 1)); + + } else { + nxt_unit_debug(ctx, "thread #%d not started", (int) (i + 1)); + } + } + + for (i = 0; i < c->threads - 1; i++) { + nxt_ruby_ctx_done(&nxt_ruby_ctxs[i]); + } + + nxt_unit_free(ctx, nxt_ruby_ctxs); +} diff --git a/src/ruby/nxt_ruby.h b/src/ruby/nxt_ruby.h index 77a65894..26430021 100644 --- a/src/ruby/nxt_ruby.h +++ b/src/ruby/nxt_ruby.h @@ -21,9 +21,13 @@ typedef struct { - nxt_unit_ctx_t *unit_ctx; + VALUE env; + VALUE io_input; + VALUE io_error; + VALUE thread; + nxt_unit_ctx_t *ctx; nxt_unit_request_info_t *req; -} nxt_ruby_run_ctx_t; +} nxt_ruby_ctx_t; VALUE nxt_ruby_stream_io_input_init(void); diff --git a/src/ruby/nxt_ruby_stream_io.c b/src/ruby/nxt_ruby_stream_io.c index cc110035..69bf289e 100644 --- a/src/ruby/nxt_ruby_stream_io.c +++ b/src/ruby/nxt_ruby_stream_io.c @@ -8,7 +8,7 @@ #include <nxt_unit.h> -static VALUE nxt_ruby_stream_io_new(VALUE class, VALUE wrap); +static VALUE nxt_ruby_stream_io_new(VALUE class, VALUE arg); static VALUE nxt_ruby_stream_io_initialize(int argc, VALUE *argv, VALUE self); static VALUE nxt_ruby_stream_io_gets(VALUE obj); static VALUE nxt_ruby_stream_io_each(VALUE obj); @@ -16,8 +16,7 @@ static VALUE nxt_ruby_stream_io_read(VALUE obj, VALUE args); static VALUE nxt_ruby_stream_io_rewind(VALUE obj); static VALUE nxt_ruby_stream_io_puts(VALUE obj, VALUE args); static VALUE nxt_ruby_stream_io_write(VALUE obj, VALUE args); -nxt_inline long nxt_ruby_stream_io_s_write(nxt_ruby_run_ctx_t *run_ctx, - VALUE val); +nxt_inline long nxt_ruby_stream_io_s_write(nxt_ruby_ctx_t *rctx, VALUE val); static VALUE nxt_ruby_stream_io_flush(VALUE obj); @@ -63,13 +62,11 @@ nxt_ruby_stream_io_error_init(void) static VALUE -nxt_ruby_stream_io_new(VALUE class, VALUE wrap) +nxt_ruby_stream_io_new(VALUE class, VALUE arg) { - VALUE self; - nxt_ruby_run_ctx_t *run_ctx; + VALUE self; - Data_Get_Struct(wrap, nxt_ruby_run_ctx_t, run_ctx); - self = Data_Wrap_Struct(class, 0, 0, run_ctx); + self = Data_Wrap_Struct(class, 0, 0, (void *) (uintptr_t) arg); rb_obj_call_init(self, 0, NULL); @@ -89,12 +86,11 @@ nxt_ruby_stream_io_gets(VALUE obj) { VALUE buf; ssize_t res; - nxt_ruby_run_ctx_t *run_ctx; + nxt_ruby_ctx_t *rctx; nxt_unit_request_info_t *req; - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); - - req = run_ctx->req; + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); + req = rctx->req; if (req->content_length == 0) { return Qnil; @@ -145,13 +141,13 @@ nxt_ruby_stream_io_each(VALUE obj) static VALUE nxt_ruby_stream_io_read(VALUE obj, VALUE args) { - VALUE buf; - long copy_size, u_size; - nxt_ruby_run_ctx_t *run_ctx; + VALUE buf; + long copy_size, u_size; + nxt_ruby_ctx_t *rctx; - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); - copy_size = run_ctx->req->content_length; + copy_size = rctx->req->content_length; if (RARRAY_LEN(args) > 0 && TYPE(RARRAY_PTR(args)[0]) == T_FIXNUM) { u_size = NUM2LONG(RARRAY_PTR(args)[0]); @@ -175,8 +171,7 @@ nxt_ruby_stream_io_read(VALUE obj, VALUE args) return Qnil; } - copy_size = nxt_unit_request_read(run_ctx->req, RSTRING_PTR(buf), - copy_size); + copy_size = nxt_unit_request_read(rctx->req, RSTRING_PTR(buf), copy_size); if (RARRAY_LEN(args) > 1 && TYPE(RARRAY_PTR(args)[1]) == T_STRING) { @@ -200,15 +195,15 @@ nxt_ruby_stream_io_rewind(VALUE obj) static VALUE nxt_ruby_stream_io_puts(VALUE obj, VALUE args) { - nxt_ruby_run_ctx_t *run_ctx; + nxt_ruby_ctx_t *rctx; if (RARRAY_LEN(args) != 1) { return Qnil; } - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); - nxt_ruby_stream_io_s_write(run_ctx, RARRAY_PTR(args)[0]); + nxt_ruby_stream_io_s_write(rctx, RARRAY_PTR(args)[0]); return Qnil; } @@ -217,23 +212,23 @@ nxt_ruby_stream_io_puts(VALUE obj, VALUE args) static VALUE nxt_ruby_stream_io_write(VALUE obj, VALUE args) { - long len; - nxt_ruby_run_ctx_t *run_ctx; + long len; + nxt_ruby_ctx_t *rctx; if (RARRAY_LEN(args) != 1) { return Qnil; } - Data_Get_Struct(obj, nxt_ruby_run_ctx_t, run_ctx); + Data_Get_Struct(obj, nxt_ruby_ctx_t, rctx); - len = nxt_ruby_stream_io_s_write(run_ctx, RARRAY_PTR(args)[0]); + len = nxt_ruby_stream_io_s_write(rctx, RARRAY_PTR(args)[0]); return LONG2FIX(len); } nxt_inline long -nxt_ruby_stream_io_s_write(nxt_ruby_run_ctx_t *run_ctx, VALUE val) +nxt_ruby_stream_io_s_write(nxt_ruby_ctx_t *rctx, VALUE val) { if (nxt_slow_path(val == Qnil)) { return 0; @@ -247,7 +242,7 @@ nxt_ruby_stream_io_s_write(nxt_ruby_run_ctx_t *run_ctx, VALUE val) } } - nxt_unit_req_error(run_ctx->req, "Ruby: %s", RSTRING_PTR(val)); + nxt_unit_req_error(rctx->req, "Ruby: %s", RSTRING_PTR(val)); return RSTRING_LEN(val); } diff --git a/src/test/nxt_http_parse_test.c b/src/test/nxt_http_parse_test.c index 9630b21c..540309c1 100644 --- a/src/test/nxt_http_parse_test.c +++ b/src/test/nxt_http_parse_test.c @@ -23,9 +23,15 @@ typedef struct { } nxt_http_parse_test_request_line_t; +typedef struct { + nxt_int_t result; + unsigned discard_unsafe_fields:1; +} nxt_http_parse_test_fields_t; + + typedef union { void *pointer; - nxt_int_t result; + nxt_http_parse_test_fields_t fields; nxt_http_parse_test_request_line_t request_line; } nxt_http_parse_test_data_t; @@ -324,10 +330,11 @@ static nxt_http_parse_test_case_t nxt_http_test_cases[] = { { nxt_string("GET / HTTP/1.1\r\n" "X-Unknown-Header: value\r\n" - "X-Good-Header: value\r\n\r\n"), + "X-Good-Header: value\r\n" + "!#$%&'*+.^_`|~: skipped\r\n\r\n"), NXT_DONE, &nxt_http_parse_test_fields, - { .result = NXT_OK } + { .fields = { NXT_OK, 1 } } }, { nxt_string("GET / HTTP/1.1\r\n" @@ -336,7 +343,14 @@ static nxt_http_parse_test_case_t nxt_http_test_cases[] = { "X-Bad-Header: value\r\n\r\n"), NXT_DONE, &nxt_http_parse_test_fields, - { .result = NXT_ERROR } + { .fields = { NXT_ERROR, 1 } } + }, + { + nxt_string("GET / HTTP/1.1\r\n" + "!#$%&'*+.^_`|~: allowed\r\n\r\n"), + NXT_DONE, + &nxt_http_parse_test_fields, + { .fields = { NXT_ERROR, 0 } } }, }; @@ -349,6 +363,10 @@ static nxt_http_field_proc_t nxt_http_test_fields[] = { { nxt_string("X-Good-Header"), &nxt_http_test_header_return, NXT_OK }, + + { nxt_string("!#$%&'*+.^_`|~"), + &nxt_http_test_header_return, + NXT_ERROR }, }; @@ -540,6 +558,10 @@ nxt_http_parse_test(nxt_thread_t *thr) return NXT_ERROR; } + if (test->handler == &nxt_http_parse_test_fields) { + rp.discard_unsafe_fields = test->data.fields.discard_unsafe_fields; + } + rc = nxt_http_parse_test_run(&rp, &test->request); if (rc != test->result) { @@ -740,7 +762,7 @@ nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, return NXT_ERROR; } - if (rp->complex_target != test->complex_target) { + if (rp->complex_target != (test->complex_target | test->quoted_target)) { nxt_log_alert(log, "http parse test case failed:\n" " - request:\n\"%V\"\n" " - complex_target: %d (expected: %d)", @@ -748,6 +770,7 @@ nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, return NXT_ERROR; } +#if 0 if (rp->quoted_target != test->quoted_target) { nxt_log_alert(log, "http parse test case failed:\n" " - request:\n\"%V\"\n" @@ -763,6 +786,7 @@ nxt_http_parse_test_request_line(nxt_http_request_parse_t *rp, request, rp->space_in_target, test->space_in_target); return NXT_ERROR; } +#endif return NXT_OK; } @@ -776,11 +800,11 @@ nxt_http_parse_test_fields(nxt_http_request_parse_t *rp, rc = nxt_http_fields_process(rp->fields, &nxt_http_test_fields_hash, NULL); - if (rc != data->result) { + if (rc != data->fields.result) { nxt_log_alert(log, "http parse test hash failed:\n" " - request:\n\"%V\"\n" " - result: %i (expected: %i)", - request, rc, data->result); + request, rc, data->fields.result); return NXT_ERROR; } diff --git a/src/test/nxt_unit_app_test.c b/src/test/nxt_unit_app_test.c index 4ecb8d6b..a5f3728c 100644 --- a/src/test/nxt_unit_app_test.c +++ b/src/test/nxt_unit_app_test.c @@ -6,6 +6,9 @@ #include <nxt_unit.h> #include <nxt_unit_request.h> #include <nxt_clang.h> +#include <pthread.h> +#include <string.h> +#include <stdlib.h> #define CONTENT_TYPE "Content-Type" @@ -28,12 +31,112 @@ #define BODY " Body:\n" -static inline char * -copy(char *p, const void *src, uint32_t len) +static int ready_handler(nxt_unit_ctx_t *ctx); +static void *worker(void *main_ctx); +static void greeting_app_request_handler(nxt_unit_request_info_t *req); +static inline char *copy(char *p, const void *src, uint32_t len); + + +static int thread_count; +static pthread_t *threads; + + +int +main(int argc, char **argv) { - memcpy(p, src, len); + int i, err; + nxt_unit_ctx_t *ctx; + nxt_unit_init_t init; - return p + len; + if (argc == 3 && strcmp(argv[1], "-t") == 0) { + thread_count = atoi(argv[2]); + } + + memset(&init, 0, sizeof(nxt_unit_init_t)); + + init.callbacks.request_handler = greeting_app_request_handler; + init.callbacks.ready_handler = ready_handler; + + ctx = nxt_unit_init(&init); + if (ctx == NULL) { + return 1; + } + + err = nxt_unit_run(ctx); + + nxt_unit_debug(ctx, "main worker finished with %d code", err); + + if (thread_count > 1) { + for (i = 0; i < thread_count - 1; i++) { + err = pthread_join(threads[i], NULL); + + if (nxt_fast_path(err == 0)) { + nxt_unit_debug(ctx, "join thread #%d", i); + + } else { + nxt_unit_alert(ctx, "pthread_join(#%d) failed: %s (%d)", + i, strerror(err), err); + } + } + + nxt_unit_free(ctx, threads); + } + + nxt_unit_done(ctx); + + nxt_unit_debug(NULL, "main worker done"); + + return 0; +} + + +static int +ready_handler(nxt_unit_ctx_t *ctx) +{ + int i, err; + + nxt_unit_debug(ctx, "ready"); + + if (!nxt_unit_is_main_ctx(ctx) || thread_count <= 1) { + return NXT_UNIT_OK; + } + + threads = nxt_unit_malloc(ctx, sizeof(pthread_t) * (thread_count - 1)); + if (threads == NULL) { + return NXT_UNIT_ERROR; + } + + for (i = 0; i < thread_count - 1; i++) { + err = pthread_create(&threads[i], NULL, worker, ctx); + if (err != 0) { + return NXT_UNIT_ERROR; + } + } + + return NXT_UNIT_OK; +} + + +static void * +worker(void *main_ctx) +{ + int rc; + nxt_unit_ctx_t *ctx; + + ctx = nxt_unit_ctx_alloc(main_ctx, NULL); + if (ctx == NULL) { + return NULL; + } + + nxt_unit_debug(ctx, "start worker"); + + rc = nxt_unit_run(ctx); + + nxt_unit_debug(ctx, "worker finished with %d code", rc); + + nxt_unit_done(ctx); + + return (void *) (intptr_t) rc; } @@ -168,24 +271,11 @@ fail: nxt_unit_request_done(req, rc); } -int -main() -{ - nxt_unit_ctx_t *ctx; - nxt_unit_init_t init; - - memset(&init, 0, sizeof(nxt_unit_init_t)); - - init.callbacks.request_handler = greeting_app_request_handler; - ctx = nxt_unit_init(&init); - if (ctx == NULL) { - return 1; - } - - nxt_unit_run(ctx); - - nxt_unit_done(ctx); +static inline char * +copy(char *p, const void *src, uint32_t len) +{ + memcpy(p, src, len); - return 0; + return p + len; } diff --git a/src/test/nxt_unit_websocket_chat.c b/src/test/nxt_unit_websocket_chat.c index 6e274722..39f8a440 100644 --- a/src/test/nxt_unit_websocket_chat.c +++ b/src/test/nxt_unit_websocket_chat.c @@ -30,7 +30,7 @@ typedef struct { static int ws_chat_root(nxt_unit_request_info_t *req); -static void ws_chat_broadcast(const void *buf, size_t size); +static void ws_chat_broadcast(const char *buf, size_t size); static const char ws_chat_index_html[]; @@ -139,18 +139,18 @@ ws_chat_root(nxt_unit_request_info_t *req) static void -ws_chat_broadcast(const void *buf, size_t size) +ws_chat_broadcast(const char *buf, size_t size) { ws_chat_request_data_t *data; nxt_unit_request_info_t *req; - nxt_unit_debug(NULL, "broadcast: %s", buf); + nxt_unit_debug(NULL, "broadcast: %*.s", (int) size, buf); nxt_queue_each(data, &ws_chat_sessions, ws_chat_request_data_t, link) { req = nxt_unit_get_request_info_from_data(data); - nxt_unit_req_debug(req, "broadcast: %s", buf); + nxt_unit_req_debug(req, "send: %*.s", (int) size, buf); nxt_unit_websocket_send(req, NXT_WEBSOCKET_OP_TEXT, 1, buf, size); } nxt_queue_loop; diff --git a/test/conftest.py b/test/conftest.py index b62264ca..3edc471d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,11 +4,13 @@ import platform import re import shutil import signal +import socket import stat import subprocess import sys import tempfile import time +from multiprocessing import Process import pytest @@ -45,6 +47,7 @@ def pytest_addoption(parser): unit_instance = {} +_processes = [] option = None @@ -66,9 +69,15 @@ def pytest_configure(config): fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, 0) +def skip_alert(*alerts): + option.skip_alerts.extend(alerts) + + def pytest_generate_tests(metafunc): cls = metafunc.cls - if not hasattr(cls, 'application_type'): + if (not hasattr(cls, 'application_type') + or cls.application_type == None + or cls.application_type == 'external'): return type = cls.application_type @@ -127,16 +136,15 @@ def pytest_sessionstart(session): break if m is None: - _print_log() + _print_log(log) exit("Unit is writing log too long") # discover available modules from unit.log for module in re.findall(r'module: ([a-zA-Z]+) (.*) ".*"$', log, re.M): - if module[0] not in option.available['modules']: - option.available['modules'][module[0]] = [module[1]] - else: - option.available['modules'][module[0]].append(module[1]) + versions = option.available['modules'].setdefault(module[0], []) + if module[1] not in versions: + versions.append(module[1]) # discover modules from check @@ -154,8 +162,26 @@ def pytest_sessionstart(session): unit_stop() + shutil.rmtree(unit_instance['temp_dir']) + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) + + +@pytest.fixture(autouse=True) +def run(request): + unit = unit_run() + option.temp_dir = unit['temp_dir'] -def setup_method(self): option.skip_alerts = [ r'read signalfd\(4\) failed', r'sendmsg.+failed', @@ -163,6 +189,40 @@ def setup_method(self): ] option.skip_sanitizer = False + yield + + # stop unit + + error = unit_stop() + + if error: + _print_log() + + assert error is None, 'stop unit' + + # stop all processes + + error = stop_processes() + + if error: + _print_log() + + assert error is None, 'stop unit' + + # check unit.log for alerts + + _check_alerts() + + # print unit.log in case of error + + if hasattr(request.node, 'rep_call') and request.node.rep_call.failed: + _print_log() + + # remove unit.log + + if not option.save_log: + shutil.rmtree(unit['temp_dir']) + def unit_run(): global unit_instance build_dir = option.current_dir + '/build' @@ -204,14 +264,6 @@ def unit_run(): _print_log() exit('Could not start unit') - # dumb (TODO: remove) - option.skip_alerts = [ - r'read signalfd\(4\) failed', - r'sendmsg.+failed', - r'recvmsg.+failed', - ] - option.skip_sanitizer = False - unit_instance['temp_dir'] = temp_dir unit_instance['log'] = temp_dir + '/unit.log' unit_instance['control_sock'] = temp_dir + '/control.unit.sock' @@ -232,12 +284,15 @@ def unit_stop(): retcode = p.wait(15) if retcode: return 'Child process terminated with code ' + str(retcode) + + except KeyboardInterrupt: + p.kill() + raise + except: p.kill() return 'Could not terminate unit' - shutil.rmtree(unit_instance['temp_dir']) - def public_dir(path): os.chmod(path, 0o777) @@ -267,11 +322,14 @@ def waitforfiles(*files): return ret -def skip_alert(*alerts): - option.skip_alerts.extend(alerts) +def _check_alerts(path=None): + if path is None: + path = unit_instance['log'] + + with open(path, 'r', encoding='utf-8', errors='ignore') as f: + log = f.read() -def _check_alerts(log): found = False alerts = re.findall(r'.+\[alert\].+', log) @@ -286,23 +344,22 @@ def _check_alerts(log): alerts = [al for al in alerts if re.search(skip, al) is None] if alerts: - _print_log(data=log) + _print_log(log) assert not alerts, 'alert(s)' if not option.skip_sanitizer: sanitizer_errors = re.findall('.+Sanitizer.+', log) if sanitizer_errors: - _print_log(data=log) + _print_log(log) assert not sanitizer_errors, 'sanitizer error(s)' if found: print('skipped.') -def _print_log(path=None, data=None): - if path is None: - path = unit_instance['log'] +def _print_log(data=None): + path = unit_instance['log'] print('Path to unit.log:\n' + path + '\n') @@ -317,6 +374,56 @@ def _print_log(path=None, data=None): sys.stdout.write(data) +def run_process(target, *args): + global _processes + + process = Process(target=target, args=args) + process.start() + + _processes.append(process) + +def stop_processes(): + if not _processes: + return + + fail = False + for process in _processes: + if process.is_alive(): + process.terminate() + process.join(timeout=15) + + if process.is_alive(): + fail = True + + if fail: + return 'Fail to stop process(es)' + + +def waitforsocket(port): + ret = False + + for i in range(50): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('127.0.0.1', port)) + ret = True + break + + except KeyboardInterrupt: + raise + + except: + sock.close() + time.sleep(0.1) + + sock.close() + + assert ret, 'socket connected' + +@pytest.fixture +def temp_dir(request): + return unit_instance['temp_dir'] + @pytest.fixture def is_unsafe(request): return request.config.getoption("--unsafe") diff --git a/test/go/ns_inspect/app.go b/test/go/ns_inspect/app.go index 4d19a796..570580e6 100644 --- a/test/go/ns_inspect/app.go +++ b/test/go/ns_inspect/app.go @@ -7,6 +7,7 @@ import ( "unit.nginx.org/go" "os" "strconv" + "io/ioutil" ) type ( @@ -26,6 +27,7 @@ type ( GID int NS NS FileExists bool + Mounts string } ) @@ -77,6 +79,11 @@ func handler(w http.ResponseWriter, r *http.Request) { out.FileExists = err == nil } + if mounts := r.Form.Get("mounts"); mounts != "" { + data, _ := ioutil.ReadFile("/proc/self/mountinfo") + out.Mounts = string(data) + } + data, err := json.Marshal(out) if err != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/test/java/threads/app.java b/test/java/threads/app.java new file mode 100644 index 00000000..d0dd3fcc --- /dev/null +++ b/test/java/threads/app.java @@ -0,0 +1,32 @@ + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet("/") +public class app extends HttpServlet +{ + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException + { + int delay = 0; + + String x_delay = request.getHeader("X-Delay"); + if (x_delay != null) { + delay = Integer.parseInt(x_delay); + } + + try { + Thread.sleep(delay * 1000); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + + response.addHeader("X-Thread", "" + Thread.currentThread().getId()); + } +} diff --git a/test/perl/threads/psgi.pl b/test/perl/threads/psgi.pl new file mode 100644 index 00000000..dce28f7d --- /dev/null +++ b/test/perl/threads/psgi.pl @@ -0,0 +1,11 @@ +my $app = sub { + my ($environ) = @_; + + sleep int($environ->{'HTTP_X_DELAY'}); + + return ['200', [ + 'Content-Length' => 0, + 'Psgi-Multithread' => $environ->{'psgi.multithread'}, + 'X-Thread' => $environ->{'psgi.input'} + ], []]; +}; diff --git a/test/php/fastcgi_finish_request/index.php b/test/php/fastcgi_finish_request/index.php new file mode 100644 index 00000000..a6211303 --- /dev/null +++ b/test/php/fastcgi_finish_request/index.php @@ -0,0 +1,11 @@ +<?php +if (!isset($_GET['skip'])) { + echo "0123"; +} + +if (!fastcgi_finish_request()) { + error_log("Error in fastcgi_finish_request"); +} + +echo "4567"; +?> diff --git a/test/pytest.ini b/test/pytest.ini index c672788a..fe86cef2 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -1,3 +1,3 @@ [pytest] -addopts = -rs -vvv +addopts = -vvv -s --print_log python_functions = test_* diff --git a/test/python/header_fields/wsgi.py b/test/python/header_fields/wsgi.py new file mode 100644 index 00000000..bd1ba0e2 --- /dev/null +++ b/test/python/header_fields/wsgi.py @@ -0,0 +1,9 @@ +def application(environ, start_response): + + h = (k for k, v in environ.items() if k.startswith('HTTP_')) + + start_response('200', [ + ('Content-Length', '0'), + ('All-Headers', ','.join(h)) + ]) + return [] diff --git a/test/python/legacy/asgi.py b/test/python/legacy/asgi.py new file mode 100644 index 00000000..f065d026 --- /dev/null +++ b/test/python/legacy/asgi.py @@ -0,0 +1,13 @@ +def application(scope): + assert scope['type'] == 'http' + + return app_http + +async def app_http(receive, send): + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + (b'content-length', b'0'), + ] + }) diff --git a/test/python/legacy_force/asgi.py b/test/python/legacy_force/asgi.py new file mode 100644 index 00000000..2e5859f2 --- /dev/null +++ b/test/python/legacy_force/asgi.py @@ -0,0 +1,17 @@ +def application(scope, receive=None, send=None): + assert scope['type'] == 'http' + + if receive == None and send == None: + return app_http + + else: + return app_http(receive, send) + +async def app_http(receive, send): + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + (b'content-length', b'0'), + ] + }) diff --git a/test/python/threads/asgi.py b/test/python/threads/asgi.py new file mode 100644 index 00000000..d51ae431 --- /dev/null +++ b/test/python/threads/asgi.py @@ -0,0 +1,27 @@ +import asyncio +import time +import threading + +async def application(scope, receive, send): + assert scope['type'] == 'http' + + headers = scope.get('headers', []) + + def get_header(n, v=None): + for h in headers: + if h[0] == n: + return h[1] + return v + + delay = float(get_header(b'x-delay', 0)) + + time.sleep(delay) + + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + (b'content-length', b'0'), + (b'x-thread', str(threading.currentThread().ident).encode()), + ] + }) diff --git a/test/python/threads/wsgi.py b/test/python/threads/wsgi.py new file mode 100644 index 00000000..1cc8ffe2 --- /dev/null +++ b/test/python/threads/wsgi.py @@ -0,0 +1,15 @@ +import time +import threading + +def application(environ, start_response): + delay = float(environ.get('HTTP_X_DELAY', 0)) + + time.sleep(delay) + + start_response('200', [ + ('Content-Length', '0'), + ('Wsgi-Multithread', str(environ['wsgi.multithread'])), + ('X-Thread', str(threading.currentThread().ident)) + ]) + + return [] diff --git a/test/ruby/threads/config.ru b/test/ruby/threads/config.ru new file mode 100644 index 00000000..2a234d0d --- /dev/null +++ b/test/ruby/threads/config.ru @@ -0,0 +1,13 @@ +app = Proc.new do |env| + delay = env['HTTP_X_DELAY'].to_f + + sleep(delay) + + ['200', { + 'Content-Length' => 0.to_s, + 'Rack-Multithread' => env['rack.multithread'].to_s, + 'X-Thread' => Thread.current.object_id.to_s + }, []] +end + +run app diff --git a/test/test_access_log.py b/test/test_access_log.py index eaba82ab..511ce6c5 100644 --- a/test/test_access_log.py +++ b/test/test_access_log.py @@ -2,6 +2,8 @@ import time import pytest +from conftest import option +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython @@ -12,7 +14,7 @@ class TestAccessLog(TestApplicationPython): super().load(script) assert 'success' in self.conf( - '"' + self.temp_dir + '/access.log"', 'access_log' + '"' + option.temp_dir + '/access.log"', 'access_log' ), 'access_log configure' def wait_for_record(self, pattern, name='access.log'): @@ -48,7 +50,7 @@ class TestAccessLog(TestApplicationPython): body='0123456789', ) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"POST / HTTP/1.1" 200 10') is not None @@ -76,7 +78,7 @@ Connection: close raw=True, ) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "Referer-1" "-"') @@ -98,7 +100,7 @@ Connection: close self.get(sock_type='ipv6') - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -110,7 +112,7 @@ Connection: close def test_access_log_unix(self): self.load('empty') - addr = self.temp_dir + '/sock' + addr = option.temp_dir + '/sock' self.conf( {"unix:" + addr: {"pass": "applications/empty"}}, 'listeners' @@ -118,7 +120,7 @@ Connection: close self.get(sock_type='unix', addr=addr) - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -138,7 +140,7 @@ Connection: close } ) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "referer-value" "-"') @@ -156,7 +158,7 @@ Connection: close } ) - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -170,7 +172,7 @@ Connection: close self.get(http_10=True) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.0" 200 0 "-" "-"') is not None @@ -185,7 +187,7 @@ Connection: close time.sleep(1) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GE" 400 0 "-" "-"') is not None @@ -198,7 +200,7 @@ Connection: close self.http(b"""GET /\n""", raw=True) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET /" 400 \d+ "-" "-"') is not None @@ -213,7 +215,7 @@ Connection: close time.sleep(1) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET /" 400 0 "-" "-"') is not None @@ -228,7 +230,7 @@ Connection: close time.sleep(1) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 400 0 "-" "-"') is not None @@ -242,7 +244,7 @@ Connection: close self.get(headers={'Connection': 'close'}) - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 400 \d+ "-" "-"') @@ -254,7 +256,7 @@ Connection: close self.get(url='/?blah&var=val') - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -270,20 +272,20 @@ Connection: close self.get(url='/delete') - self.stop() + unit_stop() assert self.search_in_log(r'/delete', 'access.log') is None, 'delete' - def test_access_log_change(self): + def test_access_log_change(self, temp_dir): self.load('empty') self.get() - self.conf('"' + self.temp_dir + '/new.log"', 'access_log') + self.conf('"' + option.temp_dir + '/new.log"', 'access_log') self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET / HTTP/1.1" 200 0 "-" "-"', 'new.log') diff --git a/test/test_asgi_application.py b/test/test_asgi_application.py index 948d9823..e90d78bc 100644 --- a/test/test_asgi_application.py +++ b/test/test_asgi_application.py @@ -4,6 +4,7 @@ from distutils.version import LooseVersion import pytest +from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython @@ -14,7 +15,7 @@ class TestASGIApplication(TestApplicationPython): load_module = 'asgi' def findall(self, pattern): - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: return re.findall(pattern, f.read()) def test_asgi_application_variables(self): @@ -136,23 +137,17 @@ custom-header: BLAH ), '204 header transfer encoding' def test_asgi_application_shm_ack_handle(self): - self.load('mirror') - # Minimum possible limit shm_limit = 10 * 1024 * 1024 - assert ( - 'success' in self.conf('{"shm": ' + str(shm_limit) + '}', - 'applications/mirror/limits') - ) + self.load('mirror', limits={"shm": shm_limit}) # Should exceed shm_limit max_body_size = 12 * 1024 * 1024 - assert ( - 'success' in self.conf('{"http":{"max_body_size": ' - + str(max_body_size) + ' }}', - 'settings') + assert 'success' in self.conf( + '{"http":{"max_body_size": ' + str(max_body_size) + ' }}', + 'settings' ) assert self.get()['status'] == 200, 'init' @@ -203,11 +198,6 @@ custom-header: BLAH assert resp['body'] == body, 'keep-alive 2' def test_asgi_keepalive_reconfigure(self): - skip_alert( - r'pthread_mutex.+failed', - r'failed to apply', - r'process \d+ exited on signal', - ) self.load('mirror') assert self.get()['status'] == 200, 'init' @@ -229,9 +219,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive open' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure' + + self.load('mirror', processes=i + 1) socks.append(sock) @@ -249,9 +238,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive request' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 2' + + self.load('mirror', processes=i + 1) for i in range(conns): resp = self.post( @@ -265,9 +253,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive close' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 3' + + self.load('mirror', processes=i + 1) def test_asgi_keepalive_reconfigure_2(self): self.load('mirror') @@ -346,11 +333,7 @@ Connection: close assert resp['status'] == 200, 'reconfigure 3' def test_asgi_process_switch(self): - self.load('delayed') - - assert 'success' in self.conf( - '2', 'applications/delayed/processes' - ), 'configure 2 processes' + self.load('delayed', processes=2) self.get( headers={ @@ -381,9 +364,7 @@ Connection: close def test_asgi_application_loading_error(self): skip_alert(r'Python failed to import module "blah"') - self.load('empty') - - assert 'success' in self.conf('"blah"', 'applications/empty/module') + self.load('empty', module="blah") assert self.get()['status'] == 503, 'loading error' @@ -400,3 +381,66 @@ Connection: close assert ( self.wait_for_record(r'\(5\) Thread: 100') is not None ), 'last thread finished' + + def test_asgi_application_threads(self): + self.load('threads', threads=2) + + socks = [] + + for i in range(2): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '3', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + time.sleep(1.0) # required to avoid greedy request reading + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['x-thread']) + + sock.close() + + assert len(socks) == len(threads), 'threads differs' + + def test_asgi_application_legacy(self): + self.load('legacy') + + resp = self.get( + headers={ + 'Host': 'localhost', + 'Content-Length': '0', + 'Connection': 'close', + }, + ) + + assert resp['status'] == 200, 'status' + + def test_asgi_application_legacy_force(self): + self.load('legacy_force', protocol='asgi') + + resp = self.get( + headers={ + 'Host': 'localhost', + 'Content-Length': '0', + 'Connection': 'close', + }, + ) + + assert resp['status'] == 200, 'status' diff --git a/test/test_asgi_lifespan.py b/test/test_asgi_lifespan.py index c37a1aae..3f29c7e7 100644 --- a/test/test_asgi_lifespan.py +++ b/test/test_asgi_lifespan.py @@ -5,6 +5,7 @@ import pytest from conftest import option from conftest import public_dir +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython @@ -34,7 +35,7 @@ class TestASGILifespan(TestApplicationPython): assert self.get()['status'] == 204 - self.stop() + unit_stop() is_startup = os.path.isfile(startup_path) is_shutdown = os.path.isfile(shutdown_path) diff --git a/test/test_asgi_websockets.py b/test/test_asgi_websockets.py index ab49b130..54984526 100644 --- a/test/test_asgi_websockets.py +++ b/test/test_asgi_websockets.py @@ -18,8 +18,6 @@ class TestASGIWebsockets(TestApplicationPython): ws = TestApplicationWebsocket() def setup_method(self): - super().setup_method() - assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_go_isolation.py b/test/test_go_isolation.py index 1e7243f6..8c4a6b9c 100644 --- a/test/test_go_isolation.py +++ b/test/test_go_isolation.py @@ -1,9 +1,13 @@ import grp import os import pwd +import shutil import pytest +from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.go import TestApplicationGo from unit.feature.isolation import TestFeatureIsolation @@ -14,11 +18,17 @@ class TestGoIsolation(TestApplicationGo): @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + unit = unit_run() + option.temp_dir = unit['temp_dir'] - return unit if not complete_check else unit.complete() + TestFeatureIsolation().check(option.available, unit['temp_dir']) + + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) + + return check if not complete_check else check() def unpriv_creds(self): nobody_uid = pwd.getpwnam('nobody').pw_uid @@ -26,21 +36,21 @@ class TestGoIsolation(TestApplicationGo): try: nogroup_gid = grp.getgrnam('nogroup').gr_gid nogroup = 'nogroup' - except: + except KeyError: nogroup_gid = grp.getgrnam('nobody').gr_gid nogroup = 'nobody' return (nobody_uid, nogroup_gid, nogroup) def isolation_key(self, key): - return key in self.available['features']['isolation'].keys() + return key in option.available['features']['isolation'].keys() def test_isolation_values(self): self.load('ns_inspect') obj = self.getjson()['body'] - for ns, ns_value in self.available['features']['isolation'].items(): + for ns, ns_value in option.available['features']['isolation'].items(): if ns.upper() in obj['NS']: assert obj['NS'][ns.upper()] == ns_value, '%s match' % ns @@ -198,7 +208,7 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson()['body'] # all but user and mnt - allns = list(self.available['features']['isolation'].keys()) + allns = list(option.available['features']['isolation'].keys()) allns.remove('user') allns.remove('mnt') @@ -206,7 +216,7 @@ class TestGoIsolation(TestApplicationGo): if ns.upper() in obj['NS']: assert ( obj['NS'][ns.upper()] - == self.available['features']['isolation'][ns] + == option.available['features']['isolation'][ns] ), ('%s match' % ns) assert obj['NS']['MNT'] != self.isolation.getns('mnt'), 'mnt set' @@ -216,13 +226,23 @@ class TestGoIsolation(TestApplicationGo): if not self.isolation_key('pid'): pytest.skip('pid namespace is not supported') - if not (is_su or self.isolation_key('unprivileged_userns_clone')): - pytest.skip('requires root or unprivileged_userns_clone') + if not is_su: + if not self.isolation_key('unprivileged_userns_clone'): + pytest.skip('unprivileged clone is not available') - self.load( - 'ns_inspect', - isolation={'namespaces': {'pid': True, 'credential': True}}, - ) + if not self.isolation_key('user'): + pytest.skip('user namespace is not supported') + + if not self.isolation_key('mnt'): + pytest.skip('mnt namespace is not supported') + + isolation = {'namespaces': {'pid': True}} + + if not is_su: + isolation['namespaces']['mount'] = True + isolation['namespaces']['credential'] = True + + self.load('ns_inspect', isolation=isolation) obj = self.getjson()['body'] @@ -230,7 +250,7 @@ class TestGoIsolation(TestApplicationGo): def test_isolation_namespace_false(self): self.load('ns_inspect') - allns = list(self.available['features']['isolation'].keys()) + allns = list(option.available['features']['isolation'].keys()) remove_list = ['unprivileged_userns_clone', 'ipc', 'cgroup'] allns = [ns for ns in allns if ns not in remove_list] @@ -256,20 +276,31 @@ class TestGoIsolation(TestApplicationGo): if ns.upper() in obj['NS']: assert ( obj['NS'][ns.upper()] - == self.available['features']['isolation'][ns] + == option.available['features']['isolation'][ns] ), ('%s match' % ns) - def test_go_isolation_rootfs_container(self): - if not self.isolation_key('unprivileged_userns_clone'): - pytest.skip('unprivileged clone is not available') + def test_go_isolation_rootfs_container(self, is_su, temp_dir): + if not is_su: + if not self.isolation_key('unprivileged_userns_clone'): + pytest.skip('unprivileged clone is not available') - if not self.isolation_key('mnt'): - pytest.skip('mnt namespace is not supported') + if not self.isolation_key('user'): + pytest.skip('user namespace is not supported') - isolation = { - 'namespaces': {'mount': True, 'credential': True}, - 'rootfs': self.temp_dir, - } + if not self.isolation_key('mnt'): + pytest.skip('mnt namespace is not supported') + + if not self.isolation_key('pid'): + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('ns_inspect', isolation=isolation) @@ -280,7 +311,7 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?file=/bin/sh')['body'] assert obj['FileExists'] == False, 'file should not exists' - def test_go_isolation_rootfs_container_priv(self, is_su): + def test_go_isolation_rootfs_container_priv(self, is_su, temp_dir): if not is_su: pytest.skip('requires root') @@ -289,7 +320,7 @@ class TestGoIsolation(TestApplicationGo): isolation = { 'namespaces': {'mount': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('ns_inspect', isolation=isolation) @@ -301,20 +332,50 @@ class TestGoIsolation(TestApplicationGo): obj = self.getjson(url='/?file=/bin/sh')['body'] assert obj['FileExists'] == False, 'file should not exists' - def test_go_isolation_rootfs_default_tmpfs(self): - if not self.isolation_key('unprivileged_userns_clone'): - pytest.skip('unprivileged clone is not available') + def test_go_isolation_rootfs_automount_tmpfs(self, is_su, temp_dir): + try: + open("/proc/self/mountinfo") + except: + pytest.skip('The system lacks /proc/self/mountinfo file') - if not self.isolation_key('mnt'): - pytest.skip('mnt namespace is not supported') + if not is_su: + if not self.isolation_key('unprivileged_userns_clone'): + pytest.skip('unprivileged clone is not available') - isolation = { - 'namespaces': {'mount': True, 'credential': True}, - 'rootfs': self.temp_dir, + if not self.isolation_key('user'): + pytest.skip('user namespace is not supported') + + if not self.isolation_key('mnt'): + pytest.skip('mnt namespace is not supported') + + if not self.isolation_key('pid'): + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } + + self.load('ns_inspect', isolation=isolation) + + obj = self.getjson(url='/?mounts=true')['body'] + + assert ( + "/ /tmp" in obj['Mounts'] and "tmpfs" in obj['Mounts'] + ), 'app has /tmp mounted on /' + + isolation['automount'] = { + 'tmpfs': False } self.load('ns_inspect', isolation=isolation) - obj = self.getjson(url='/?file=/tmp')['body'] + obj = self.getjson(url='/?mounts=true')['body'] - assert obj['FileExists'] == True, 'app has /tmp' + assert ( + "/ /tmp" not in obj['Mounts'] and "tmpfs" not in obj['Mounts'] + ), 'app has no /tmp mounted' diff --git a/test/test_go_isolation_rootfs.py b/test/test_go_isolation_rootfs.py index d8e177b1..1cc59c67 100644 --- a/test/test_go_isolation_rootfs.py +++ b/test/test_go_isolation_rootfs.py @@ -8,7 +8,7 @@ from unit.applications.lang.go import TestApplicationGo class TestGoIsolationRootfs(TestApplicationGo): prerequisites = {'modules': {'go': 'all'}} - def test_go_isolation_rootfs_chroot(self, is_su): + def test_go_isolation_rootfs_chroot(self, is_su, temp_dir): if not is_su: pytest.skip('requires root') @@ -16,7 +16,7 @@ class TestGoIsolationRootfs(TestApplicationGo): pytest.skip('chroot tests not supported on OSX') isolation = { - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('ns_inspect', isolation=isolation) diff --git a/test/test_http_header.py b/test/test_http_header.py index 8381a0d9..fdb557cf 100644 --- a/test/test_http_header.py +++ b/test/test_http_header.py @@ -154,54 +154,58 @@ Connection: close def test_http_header_field_leading_sp(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - ' Custom-Header': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field leading sp' + assert ( + self.get( + headers={ + 'Host': 'localhost', + ' Custom-Header': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field leading sp' def test_http_header_field_leading_htab(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - '\tCustom-Header': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field leading htab' + assert ( + self.get( + headers={ + 'Host': 'localhost', + '\tCustom-Header': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field leading htab' def test_http_header_field_trailing_sp(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - 'Custom-Header ': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field trailing sp' + assert ( + self.get( + headers={ + 'Host': 'localhost', + 'Custom-Header ': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field trailing sp' def test_http_header_field_trailing_htab(self): self.load('empty') - resp = self.get( - headers={ - 'Host': 'localhost', - 'Custom-Header\t': 'blah', - 'Connection': 'close', - } - ) - - assert resp['status'] == 400, 'field trailing htab' + assert ( + self.get( + headers={ + 'Host': 'localhost', + 'Custom-Header\t': 'blah', + 'Connection': 'close', + } + )['status'] + == 400 + ), 'field trailing htab' def test_http_header_content_length_big(self): self.load('empty') @@ -427,3 +431,41 @@ Connection: close )['status'] == 400 ), 'Host multiple fields' + + def test_http_discard_unsafe_fields(self): + self.load('header_fields') + + def check_status(header): + resp = self.get( + headers={ + 'Host': 'localhost', + header: 'blah', + 'Connection': 'close', + } + ) + + assert resp['status'] == 200 + return resp + + resp = check_status("!Custom-Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] + + resp = check_status("Custom_Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] + + assert 'success' in self.conf( + {'http': {'discard_unsafe_fields': False}}, 'settings', + ) + + resp = check_status("!#$%&'*+.^`|~Custom_Header") + assert 'CUSTOM' in resp['headers']['All-Headers'] + + assert 'success' in self.conf( + {'http': {'discard_unsafe_fields': True}}, 'settings', + ) + + resp = check_status("!Custom-Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] + + resp = check_status("Custom_Header") + assert 'CUSTOM' not in resp['headers']['All-Headers'] diff --git a/test/test_java_application.py b/test/test_java_application.py index afcdf651..41345e87 100644 --- a/test/test_java_application.py +++ b/test/test_java_application.py @@ -11,7 +11,7 @@ from unit.applications.lang.java import TestApplicationJava class TestJavaApplication(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} - def test_java_conf_error(self): + def test_java_conf_error(self, temp_dir): skip_alert( r'realpath.*failed', r'failed to apply new conf', @@ -25,18 +25,18 @@ class TestJavaApplication(TestApplicationJava): "type": "java", "processes": 1, "working_directory": option.test_dir + "/java/empty", - "webapp": self.temp_dir + "/java", - "unit_jars": self.temp_dir + "/no_such_dir", + "webapp": temp_dir + "/java", + "unit_jars": temp_dir + "/no_such_dir", } }, } ), 'conf error' - def test_java_war(self): + def test_java_war(self, temp_dir): self.load('empty_war') assert 'success' in self.conf( - '"' + self.temp_dir + '/java/empty.war"', + '"' + temp_dir + '/java/empty.war"', '/config/applications/empty_war/webapp', ), 'configure war' @@ -969,11 +969,11 @@ class TestJavaApplication(TestApplicationJava): ), 'set date header' assert headers['X-Get-Date'] == date, 'get date header' - def test_java_application_multipart(self): + def test_java_application_multipart(self, temp_dir): self.load('multipart') reldst = '/uploads' - fulldst = self.temp_dir + reldst + fulldst = temp_dir + reldst os.mkdir(fulldst) public_dir(fulldst) @@ -1012,3 +1012,44 @@ class TestJavaApplication(TestApplicationJava): ) is not None ), 'file created' + + def test_java_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + time.sleep(0.25) # required to avoid greedy request reading + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + sock.close() + + assert len(socks) == len(threads), 'threads differs' diff --git a/test/test_java_isolation_rootfs.py b/test/test_java_isolation_rootfs.py index f0f04df1..02d35a62 100644 --- a/test/test_java_isolation_rootfs.py +++ b/test/test_java_isolation_rootfs.py @@ -11,14 +11,12 @@ class TestJavaIsolationRootfs(TestApplicationJava): prerequisites = {'modules': {'java': 'all'}} def setup_method(self, is_su): - super().setup_method() - if not is_su: return - os.makedirs(self.temp_dir + '/jars') - os.makedirs(self.temp_dir + '/tmp') - os.chmod(self.temp_dir + '/tmp', 0o777) + os.makedirs(option.temp_dir + '/jars') + os.makedirs(option.temp_dir + '/tmp') + os.chmod(option.temp_dir + '/tmp', 0o777) try: process = subprocess.Popen( @@ -26,15 +24,18 @@ class TestJavaIsolationRootfs(TestApplicationJava): "mount", "--bind", option.current_dir + "/build", - self.temp_dir + "/jars", + option.temp_dir + "/jars", ], stderr=subprocess.STDOUT, ) process.communicate() + except KeyboardInterrupt: + raise + except: - pytest.fail('Cann\'t run mount process.') + pytest.fail('Can\'t run mount process.') def teardown_method(self, is_su): if not is_su: @@ -42,24 +43,24 @@ class TestJavaIsolationRootfs(TestApplicationJava): try: process = subprocess.Popen( - ["umount", "--lazy", self.temp_dir + "/jars"], + ["umount", "--lazy", option.temp_dir + "/jars"], stderr=subprocess.STDOUT, ) process.communicate() - except: - pytest.fail('Cann\'t run mount process.') + except KeyboardInterrupt: + raise - # super teardown must happen after unmount to avoid deletion of /build - super().teardown_method() + except: + pytest.fail('Can\'t run mount process.') - def test_java_isolation_rootfs_chroot_war(self, is_su): + def test_java_isolation_rootfs_chroot_war(self, is_su, temp_dir): if not is_su: pytest.skip('require root') isolation = { - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } self.load('empty_war', isolation=isolation) diff --git a/test/test_java_websockets.py b/test/test_java_websockets.py index 7e6d82e8..7586d4aa 100644 --- a/test/test_java_websockets.py +++ b/test/test_java_websockets.py @@ -15,8 +15,6 @@ class TestJavaWebsockets(TestApplicationJava): ws = TestApplicationWebsocket() def setup_method(self): - super().setup_method() - assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_node_application.py b/test/test_node_application.py index a0b882f3..c8c3a444 100644 --- a/test/test_node_application.py +++ b/test/test_node_application.py @@ -139,11 +139,11 @@ class TestNodeApplication(TestApplicationNode): assert self.get()['body'] == 'buffer', 'write buffer' - def test_node_application_write_callback(self): + def test_node_application_write_callback(self, temp_dir): self.load('write_callback') assert self.get()['body'] == 'helloworld', 'write callback order' - assert waitforfiles(self.temp_dir + '/node/callback'), 'write callback' + assert waitforfiles(temp_dir + '/node/callback'), 'write callback' def test_node_application_write_before_write_head(self): self.load('write_before_write_head') @@ -222,7 +222,7 @@ class TestNodeApplication(TestApplicationNode): assert 'X-Header' not in headers, 'insensitive' assert 'X-header' not in headers, 'insensitive 2' - def test_node_application_promise_handler(self): + def test_node_application_promise_handler(self, temp_dir): self.load('promise_handler') assert ( @@ -236,7 +236,7 @@ class TestNodeApplication(TestApplicationNode): )['status'] == 200 ), 'promise handler request' - assert waitforfiles(self.temp_dir + '/node/callback'), 'promise handler' + assert waitforfiles(temp_dir + '/node/callback'), 'promise handler' def test_node_application_promise_handler_write_after_end(self): self.load('promise_handler') @@ -254,7 +254,7 @@ class TestNodeApplication(TestApplicationNode): == 200 ), 'promise handler request write after end' - def test_node_application_promise_end(self): + def test_node_application_promise_end(self, temp_dir): self.load('promise_end') assert ( @@ -268,9 +268,9 @@ class TestNodeApplication(TestApplicationNode): )['status'] == 200 ), 'promise end request' - assert waitforfiles(self.temp_dir + '/node/callback'), 'promise end' + assert waitforfiles(temp_dir + '/node/callback'), 'promise end' - def test_node_application_promise_multiple_calls(self): + def test_node_application_promise_multiple_calls(self, temp_dir): self.load('promise_handler') self.post( @@ -283,7 +283,7 @@ class TestNodeApplication(TestApplicationNode): ) assert waitforfiles( - self.temp_dir + '/node/callback1' + temp_dir + '/node/callback1' ), 'promise first call' self.post( @@ -296,7 +296,7 @@ class TestNodeApplication(TestApplicationNode): ) assert waitforfiles( - self.temp_dir + '/node/callback2' + temp_dir + '/node/callback2' ), 'promise second call' @pytest.mark.skip('not yet') diff --git a/test/test_node_websockets.py b/test/test_node_websockets.py index 6a6b7f2d..7b65b5c1 100644 --- a/test/test_node_websockets.py +++ b/test/test_node_websockets.py @@ -15,8 +15,6 @@ class TestNodeWebsockets(TestApplicationNode): ws = TestApplicationWebsocket() def setup_method(self): - super().setup_method() - assert 'success' in self.conf( {'http': {'websocket': {'keepalive_interval': 0}}}, 'settings' ), 'clear keepalive_interval' diff --git a/test/test_perl_application.py b/test/test_perl_application.py index 78e32a43..78f2dd90 100644 --- a/test/test_perl_application.py +++ b/test/test_perl_application.py @@ -3,6 +3,7 @@ import re import pytest from conftest import skip_alert +from conftest import unit_stop from unit.applications.lang.perl import TestApplicationPerl @@ -119,7 +120,7 @@ class TestPerlApplication(TestApplicationPerl): assert self.get()['body'] == '1', 'errors result' - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application') @@ -237,3 +238,44 @@ class TestPerlApplication(TestApplicationPerl): assert resp['status'] == 200, 'status' assert resp['body'] == 'Hello World!', 'body' + + def test_perl_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + assert resp['headers']['Psgi-Multithread'] == '1', 'multithread' + + sock.close() + + assert len(socks) == len(threads), 'threads differs' diff --git a/test/test_php_application.py b/test/test_php_application.py index 063d3e0c..578de0b7 100644 --- a/test/test_php_application.py +++ b/test/test_php_application.py @@ -6,6 +6,7 @@ import time import pytest from conftest import option +from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP class TestPHPApplication(TestApplicationPHP): @@ -93,6 +94,32 @@ class TestPHPApplication(TestApplicationPHP): assert resp['status'] == 200, 'query string empty status' assert resp['headers']['Query-String'] == '', 'query string empty' + def test_php_application_fastcgi_finish_request(self, temp_dir): + self.load('fastcgi_finish_request') + + assert self.get()['body'] == '0123' + + unit_stop() + + with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: + errs = re.findall(r'Error in fastcgi_finish_request', f.read()) + + assert len(errs) == 0, 'no error' + + def test_php_application_fastcgi_finish_request_2(self, temp_dir): + self.load('fastcgi_finish_request') + + resp = self.get(url='/?skip') + assert resp['status'] == 200 + assert resp['body'] == '' + + unit_stop() + + with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: + errs = re.findall(r'Error in fastcgi_finish_request', f.read()) + + assert len(errs) == 0, 'no error' + def test_php_application_query_string_absent(self): self.load('query_string') @@ -444,7 +471,7 @@ class TestPHPApplication(TestApplicationPHP): r'012345', self.get()['body'] ), 'disable_classes before' - def test_php_application_error_log(self): + def test_php_application_error_log(self, temp_dir): self.load('error_log') assert self.get()['status'] == 200, 'status' @@ -453,13 +480,13 @@ class TestPHPApplication(TestApplicationPHP): assert self.get()['status'] == 200, 'status 2' - self.stop() + unit_stop() pattern = r'\d{4}\/\d\d\/\d\d\s\d\d:.+\[notice\].+Error in application' assert self.wait_for_record(pattern) is not None, 'errors print' - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(temp_dir + '/unit.log', 'r', errors='ignore') as f: errs = re.findall(pattern, f.read()) assert len(errs) == 2, 'error_log count' @@ -507,12 +534,12 @@ class TestPHPApplication(TestApplicationPHP): assert resp['status'] == 200, 'status' assert resp['body'] != '', 'body not empty' - def test_php_application_extension_check(self): + def test_php_application_extension_check(self, temp_dir): self.load('phpinfo') assert self.get(url='/index.wrong')['status'] != 200, 'status' - new_root = self.temp_dir + "/php" + new_root = temp_dir + "/php" os.mkdir(new_root) shutil.copy(option.test_dir + '/php/phpinfo/index.wrong', new_root) diff --git a/test/test_php_isolation.py b/test/test_php_isolation.py index 8ab3419a..cc660e04 100644 --- a/test/test_php_isolation.py +++ b/test/test_php_isolation.py @@ -1,6 +1,10 @@ +import shutil + import pytest from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.php import TestApplicationPHP from unit.feature.isolation import TestFeatureIsolation @@ -8,67 +12,85 @@ from unit.feature.isolation import TestFeatureIsolation class TestPHPIsolation(TestApplicationPHP): prerequisites = {'modules': {'php': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + unit = unit_run() + option.temp_dir = unit['temp_dir'] - return unit if not complete_check else unit.complete() + TestFeatureIsolation().check(option.available, unit['temp_dir']) - def test_php_isolation_rootfs(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') + return check if not complete_check else check() - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') + def test_php_isolation_rootfs(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() + if not is_su: if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') - isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': option.test_dir, - } + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('phpinfo', isolation=isolation) assert 'success' in self.conf( - '"/php/phpinfo"', 'applications/phpinfo/root' + '"/app/php/phpinfo"', 'applications/phpinfo/root' ) assert 'success' in self.conf( - '"/php/phpinfo"', 'applications/phpinfo/working_directory' + '"/app/php/phpinfo"', 'applications/phpinfo/working_directory' ) assert self.get()['status'] == 200, 'empty rootfs' - def test_php_isolation_rootfs_extensions(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + def test_php_isolation_rootfs_extensions(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') - isolation = { - 'rootfs': option.test_dir, - 'namespaces': {'credential': not is_su, 'mount': not is_su}, - } + isolation = {'rootfs': temp_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('list-extensions', isolation=isolation) assert 'success' in self.conf( - '"/php/list-extensions"', 'applications/list-extensions/root' + '"/app/php/list-extensions"', 'applications/list-extensions/root' ) assert 'success' in self.conf( @@ -77,7 +99,7 @@ class TestPHPIsolation(TestApplicationPHP): ) assert 'success' in self.conf( - '"/php/list-extensions"', + '"/app/php/list-extensions"', 'applications/list-extensions/working_directory', ) diff --git a/test/test_proxy.py b/test/test_proxy.py index d02c96a7..be3e93fd 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -5,7 +5,9 @@ import time import pytest from conftest import option +from conftest import run_process from conftest import skip_alert +from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython @@ -60,10 +62,8 @@ Content-Length: 10 return self.post(*args, http_10=True, **kwargs) def setup_method(self): - super().setup_method() - - self.run_process(self.run_server, self.SERVER_PORT) - self.waitforsocket(self.SERVER_PORT) + run_process(self.run_server, self.SERVER_PORT) + waitforsocket(self.SERVER_PORT) assert 'success' in self.conf( { @@ -346,8 +346,8 @@ Content-Length: 10 assert self.get_http10()['status'] == 200, 'status' - def test_proxy_unix(self): - addr = self.temp_dir + '/sock' + def test_proxy_unix(self, temp_dir): + addr = temp_dir + '/sock' assert 'success' in self.conf( { diff --git a/test/test_proxy_chunked.py b/test/test_proxy_chunked.py index 26023617..ae2228fa 100644 --- a/test/test_proxy_chunked.py +++ b/test/test_proxy_chunked.py @@ -3,6 +3,9 @@ import select import socket import time +from conftest import option +from conftest import run_process +from conftest import waitforsocket from unit.applications.lang.python import TestApplicationPython @@ -82,10 +85,8 @@ class TestProxyChunked(TestApplicationPython): return self.get(*args, http_10=True, **kwargs) def setup_method(self): - super().setup_method() - - self.run_process(self.run_server, self.SERVER_PORT, self.temp_dir) - self.waitforsocket(self.SERVER_PORT) + run_process(self.run_server, self.SERVER_PORT, option.temp_dir) + waitforsocket(self.SERVER_PORT) assert 'success' in self.conf( { diff --git a/test/test_python_application.py b/test/test_python_application.py index 3e27a24c..83b0c8f4 100644 --- a/test/test_python_application.py +++ b/test/test_python_application.py @@ -5,7 +5,9 @@ import time import pytest +from conftest import option from conftest import skip_alert +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython @@ -13,7 +15,7 @@ class TestPythonApplication(TestApplicationPython): prerequisites = {'modules': {'python': 'all'}} def findall(self, pattern): - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: return re.findall(pattern, f.read()) def test_python_application_variables(self): @@ -156,7 +158,7 @@ custom-header: BLAH self.conf({"listeners": {}, "applications": {}}) - self.stop() + unit_stop() assert ( self.wait_for_record(r'RuntimeError') is not None @@ -195,11 +197,6 @@ custom-header: BLAH assert resp['body'] == body, 'keep-alive 2' def test_python_keepalive_reconfigure(self): - skip_alert( - r'pthread_mutex.+failed', - r'failed to apply', - r'process \d+ exited on signal', - ) self.load('mirror') assert self.get()['status'] == 200, 'init' @@ -221,9 +218,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive open' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure' + + self.load('mirror', processes=i + 1) socks.append(sock) @@ -241,9 +237,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive request' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 2' + + self.load('mirror', processes=i + 1) for i in range(conns): resp = self.post( @@ -257,9 +252,8 @@ custom-header: BLAH ) assert resp['body'] == body, 'keep-alive close' - assert 'success' in self.conf( - str(i + 1), 'applications/mirror/processes' - ), 'reconfigure 3' + + self.load('mirror', processes=i + 1) def test_python_keepalive_reconfigure_2(self): self.load('mirror') @@ -344,16 +338,12 @@ Connection: close self.conf({"listeners": {}, "applications": {}}) - self.stop() + unit_stop() assert self.wait_for_record(r'At exit called\.') is not None, 'atexit' def test_python_process_switch(self): - self.load('delayed') - - assert 'success' in self.conf( - '2', 'applications/delayed/processes' - ), 'configure 2 processes' + self.load('delayed', processes=2) self.get( headers={ @@ -507,7 +497,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application\.') @@ -539,9 +529,7 @@ last line: 987654321 def test_python_application_loading_error(self): skip_alert(r'Python failed to import module "blah"') - self.load('empty') - - assert 'success' in self.conf('"blah"', 'applications/empty/module') + self.load('empty', module="blah") assert self.get()['status'] == 503, 'loading error' @@ -550,7 +538,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert self.wait_for_record(r'Close called\.') is not None, 'close' @@ -559,7 +547,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'Close called\.') is not None @@ -570,7 +558,7 @@ last line: 987654321 self.get() - self.stop() + unit_stop() assert ( self.wait_for_record( @@ -742,7 +730,7 @@ last line: 987654321 try: group_id = grp.getgrnam(group).gr_gid - except: + except KeyError: group = 'nogroup' group_id = grp.getgrnam(group).gr_gid @@ -787,7 +775,7 @@ last line: 987654321 try: grp.getgrnam(group) group = True - except: + except KeyError: group = False if group: @@ -809,24 +797,47 @@ last line: 987654321 assert self.get()['status'] == 204, 'default application response' - assert 'success' in self.conf( - '"app"', 'applications/callable/callable' - ) + self.load('callable', callable="app") assert self.get()['status'] == 200, 'callable response' - assert 'success' in self.conf( - '"blah"', 'applications/callable/callable' - ) + self.load('callable', callable="blah") assert self.get()['status'] not in [200, 204], 'callable response inv' - assert 'success' in self.conf( - '"app"', 'applications/callable/callable' - ) + def test_python_application_threads(self): + self.load('threads', threads=4) + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) - assert self.get()['status'] == 200, 'callable response 2' + assert resp['headers']['Wsgi-Multithread'] == 'True', 'multithread' - assert 'success' in self.conf_delete('applications/callable/callable') + sock.close() - assert self.get()['status'] == 204, 'default response 2' + assert len(socks) == len(threads), 'threads differs' diff --git a/test/test_python_isolation.py b/test/test_python_isolation.py index ac678103..1a157528 100644 --- a/test/test_python_isolation.py +++ b/test/test_python_isolation.py @@ -1,5 +1,10 @@ +import shutil + import pytest +from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.python import TestApplicationPython from unit.feature.isolation import TestFeatureIsolation @@ -7,48 +12,55 @@ from unit.feature.isolation import TestFeatureIsolation class TestPythonIsolation(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + unit = unit_run() + option.temp_dir = unit['temp_dir'] - return unit if not complete_check else unit.complete() + TestFeatureIsolation().check(option.available, unit['temp_dir']) - def test_python_isolation_rootfs(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') + return check if not complete_check else check() - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') + def test_python_isolation_rootfs(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() + if not is_su: if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') - isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': self.temp_dir, - } + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') - self.load('empty', isolation=isolation) + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': temp_dir} - assert self.get()['status'] == 200, 'python rootfs' + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } self.load('ns_inspect', isolation=isolation) assert ( - self.getjson(url='/?path=' + self.temp_dir)['body']['FileExists'] + self.getjson(url='/?path=' + temp_dir)['body']['FileExists'] == False ), 'temp_dir does not exists in rootfs' assert ( self.getjson(url='/?path=/proc/self')['body']['FileExists'] - == False + == True ), 'no /proc/self' assert ( @@ -66,25 +78,34 @@ class TestPythonIsolation(TestApplicationPython): ret['body']['FileExists'] == True ), 'application exists in rootfs' - def test_python_isolation_rootfs_no_language_deps(self, is_su): - isolation_features = self.available['features']['isolation'].keys() - - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') + def test_python_isolation_rootfs_no_language_deps(self, is_su, temp_dir): + isolation_features = option.available['features']['isolation'].keys() if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') - if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, 'automount': {'language_deps': False} } + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } + self.load('empty', isolation=isolation) assert (self.get()['status'] != 200), 'disabled language_deps' diff --git a/test/test_python_isolation_chroot.py b/test/test_python_isolation_chroot.py index 315fee9f..8018d5b9 100644 --- a/test/test_python_isolation_chroot.py +++ b/test/test_python_isolation_chroot.py @@ -7,28 +7,24 @@ from unit.feature.isolation import TestFeatureIsolation class TestPythonIsolation(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} - def test_python_isolation_chroot(self, is_su): + def test_python_isolation_chroot(self, is_su, temp_dir): if not is_su: pytest.skip('requires root') isolation = { - 'rootfs': self.temp_dir, + 'rootfs': temp_dir, } - self.load('empty', isolation=isolation) - - assert self.get()['status'] == 200, 'python chroot' - self.load('ns_inspect', isolation=isolation) assert ( - self.getjson(url='/?path=' + self.temp_dir)['body']['FileExists'] + self.getjson(url='/?path=' + temp_dir)['body']['FileExists'] == False ), 'temp_dir does not exists in rootfs' assert ( self.getjson(url='/?path=/proc/self')['body']['FileExists'] - == False + == True ), 'no /proc/self' assert ( diff --git a/test/test_python_procman.py b/test/test_python_procman.py index 8eccae3e..ff914fc8 100644 --- a/test/test_python_procman.py +++ b/test/test_python_procman.py @@ -4,6 +4,7 @@ import time import pytest +from conftest import option from unit.applications.lang.python import TestApplicationPython @@ -11,9 +12,7 @@ class TestPythonProcman(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} def setup_method(self): - super().setup_method() - - self.app_name = "app-" + self.temp_dir.split('/')[-1] + self.app_name = "app-" + option.temp_dir.split('/')[-1] self.app_proc = 'applications/' + self.app_name + '/processes' self.load('empty', self.app_name) diff --git a/test/test_respawn.py b/test/test_respawn.py index 18b9d535..09a806d4 100644 --- a/test/test_respawn.py +++ b/test/test_respawn.py @@ -2,6 +2,7 @@ import re import subprocess import time +from conftest import option from conftest import skip_alert from unit.applications.lang.python import TestApplicationPython @@ -13,9 +14,7 @@ class TestRespawn(TestApplicationPython): PATTERN_CONTROLLER = 'unit: controller' def setup_method(self): - super().setup_method() - - self.app_name = "app-" + self.temp_dir.split('/')[-1] + self.app_name = "app-" + option.temp_dir.split('/')[-1] self.load('empty', self.app_name) diff --git a/test/test_return.py b/test/test_return.py index 64050022..2f7b7ae4 100644 --- a/test/test_return.py +++ b/test/test_return.py @@ -7,8 +7,6 @@ class TestReturn(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - self._load_conf( { "listeners": {"*:7080": {"pass": "routes"}}, diff --git a/test/test_routing.py b/test/test_routing.py index 2b528435..83852273 100644 --- a/test/test_routing.py +++ b/test/test_routing.py @@ -10,8 +10,6 @@ class TestRouting(TestApplicationProto): prerequisites = {'modules': {'python': 'any'}} def setup_method(self): - super().setup_method() - assert 'success' in self.conf( { "listeners": {"*:7080": {"pass": "routes"}}, @@ -232,6 +230,48 @@ class TestRouting(TestApplicationProto): assert self.get(url='/aBCaBbc')['status'] == 200 assert self.get(url='/ABc')['status'] == 404 + def test_routes_empty_regex(self): + self.route_match({"uri":"~"}) + assert self.get(url='/')['status'] == 200, 'empty regexp' + assert self.get(url='/anything')['status'] == 200, '/anything' + + self.route_match({"uri":"!~"}) + assert self.get(url='/')['status'] == 404, 'empty regexp 2' + assert self.get(url='/nothing')['status'] == 404, '/nothing' + + def test_routes_bad_regex(self): + assert 'error' in self.route( + {"match": {"uri": "~/bl[ah"}, "action": {"return": 200}} + ), 'bad regex' + + status = self.route( + {"match": {"uri": "~(?R)?z"}, "action": {"return": 200}} + ) + if 'error' not in status: + assert self.get(url='/nothing_z')['status'] == 500, '/nothing_z' + + status = self.route( + {"match": {"uri": "~((?1)?z)"}, "action": {"return": 200}} + ) + if 'error' not in status: + assert self.get(url='/nothing_z')['status'] == 500, '/nothing_z' + + def test_routes_match_regex_case_sensitive(self): + self.route_match({"uri": "~/bl[ah]"}) + + assert self.get(url='/rlah')['status'] == 404, '/rlah' + assert self.get(url='/blah')['status'] == 200, '/blah' + assert self.get(url='/blh')['status'] == 200, '/blh' + assert self.get(url='/BLAH')['status'] == 404, '/BLAH' + + def test_routes_match_regex_negative_case_sensitive(self): + self.route_match({"uri": "!~/bl[ah]"}) + + assert self.get(url='/rlah')['status'] == 200, '/rlah' + assert self.get(url='/blah')['status'] == 404, '/blah' + assert self.get(url='/blh')['status'] == 404, '/blh' + assert self.get(url='/BLAH')['status'] == 200, '/BLAH' + def test_routes_pass_encode(self): def check_pass(path, name): assert 'success' in self.conf( @@ -417,7 +457,7 @@ class TestRouting(TestApplicationProto): [{"action": {"pass": "upstreams/blah"}}], 'routes' ), 'route pass upstreams invalid' - def test_routes_action_unique(self): + def test_routes_action_unique(self, temp_dir): assert 'success' in self.conf( { "listeners": { @@ -437,7 +477,7 @@ class TestRouting(TestApplicationProto): ) assert 'error' in self.conf( - {"proxy": "http://127.0.0.1:7081", "share": self.temp_dir}, + {"proxy": "http://127.0.0.1:7081", "share": temp_dir}, 'routes/0/action', ), 'proxy share' assert 'error' in self.conf( @@ -445,7 +485,7 @@ class TestRouting(TestApplicationProto): 'routes/0/action', ), 'proxy pass' assert 'error' in self.conf( - {"share": self.temp_dir, "pass": "applications/app"}, + {"share": temp_dir, "pass": "applications/app"}, 'routes/0/action', ), 'share pass' @@ -1665,8 +1705,8 @@ class TestRouting(TestApplicationProto): assert self.get(sock_type='ipv6')['status'] == 200, '0' assert self.get(port=7081)['status'] == 404, '0 ipv4' - def test_routes_source_unix(self): - addr = self.temp_dir + '/sock' + def test_routes_source_unix(self, temp_dir): + addr = temp_dir + '/sock' assert 'success' in self.conf( {"unix:" + addr: {"pass": "routes"}}, 'listeners' diff --git a/test/test_ruby_application.py b/test/test_ruby_application.py index f84935f8..e42fb97f 100644 --- a/test/test_ruby_application.py +++ b/test/test_ruby_application.py @@ -3,6 +3,7 @@ import re import pytest from conftest import skip_alert +from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby @@ -175,7 +176,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application') @@ -187,7 +188,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+1234567890') is not None @@ -198,7 +199,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Error in application') @@ -215,7 +216,7 @@ class TestRubyApplication(TestApplicationRuby): self.get() - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+1234567890') is not None @@ -228,7 +229,7 @@ class TestRubyApplication(TestApplicationRuby): self.conf({"listeners": {}, "applications": {}}) - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+At exit called\.') is not None @@ -289,7 +290,7 @@ class TestRubyApplication(TestApplicationRuby): assert self.get()['status'] == 500, 'body each error status' - self.stop() + unit_stop() assert ( self.wait_for_record(r'\[error\].+Failed to run ruby script') @@ -350,3 +351,44 @@ class TestRubyApplication(TestApplicationRuby): assert len(headers['X-Release-Date']) > 0, 'RUBY_RELEASE_DATE' assert len(headers['X-Revision']) > 0, 'RUBY_REVISION' assert len(headers['X-Version']) > 0, 'RUBY_VERSION' + + def test_ruby_application_threads(self): + self.load('threads') + + assert 'success' in self.conf( + '4', 'applications/threads/threads' + ), 'configure 4 threads' + + socks = [] + + for i in range(4): + (_, sock) = self.get( + headers={ + 'Host': 'localhost', + 'X-Delay': '2', + 'Connection': 'close', + }, + no_recv=True, + start=True, + ) + + socks.append(sock) + + threads = set() + + for sock in socks: + resp = self.recvall(sock).decode('utf-8') + + self.log_in(resp) + + resp = self._resp_to_dict(resp) + + assert resp['status'] == 200, 'status' + + threads.add(resp['headers']['X-Thread']) + + assert resp['headers']['Rack-Multithread'] == 'true', 'multithread' + + sock.close() + + assert len(socks) == len(threads), 'threads differs' diff --git a/test/test_ruby_isolation.py b/test/test_ruby_isolation.py index 13ca0e16..69e25de9 100644 --- a/test/test_ruby_isolation.py +++ b/test/test_ruby_isolation.py @@ -1,7 +1,10 @@ +import shutil import pytest from conftest import option +from conftest import unit_run +from conftest import unit_stop from unit.applications.lang.ruby import TestApplicationRuby from unit.feature.isolation import TestFeatureIsolation @@ -9,33 +12,63 @@ from unit.feature.isolation import TestFeatureIsolation class TestRubyIsolation(TestApplicationRuby): prerequisites = {'modules': {'ruby': 'any'}, 'features': ['isolation']} - isolation = TestFeatureIsolation() - @classmethod def setup_class(cls, complete_check=True): - unit = super().setup_class(complete_check=False) + check = super().setup_class(complete_check=False) - TestFeatureIsolation().check(cls.available, unit.temp_dir) + unit = unit_run() + option.temp_dir = unit['temp_dir'] - return unit if not complete_check else unit.complete() + TestFeatureIsolation().check(option.available, unit['temp_dir']) - def test_ruby_isolation_rootfs(self, is_su): - isolation_features = self.available['features']['isolation'].keys() + assert unit_stop() is None + shutil.rmtree(unit['temp_dir']) - if 'mnt' not in isolation_features: - pytest.skip('requires mnt ns') + return check if not complete_check else check() - if not is_su: - if 'user' not in isolation_features: - pytest.skip('requires unprivileged userns or root') + def test_ruby_isolation_rootfs_mount_namespace(self, is_su): + isolation_features = option.available['features']['isolation'].keys() + if not is_su: if not 'unprivileged_userns_clone' in isolation_features: pytest.skip('requires unprivileged userns or root') - isolation = { - 'namespaces': {'credential': not is_su, 'mount': True}, - 'rootfs': option.test_dir, - } + if 'user' not in isolation_features: + pytest.skip('user namespace is not supported') + + if 'mnt' not in isolation_features: + pytest.skip('mnt namespace is not supported') + + if 'pid' not in isolation_features: + pytest.skip('pid namespace is not supported') + + isolation = {'rootfs': option.test_dir} + + if not is_su: + isolation['namespaces'] = { + 'mount': True, + 'credential': True, + 'pid': True + } + + self.load('status_int', isolation=isolation) + + assert 'success' in self.conf( + '"/ruby/status_int/config.ru"', 'applications/status_int/script', + ) + + assert 'success' in self.conf( + '"/ruby/status_int"', 'applications/status_int/working_directory', + ) + + assert self.get()['status'] == 200, 'status int' + + def test_ruby_isolation_rootfs(self, is_su): + if not is_su: + pytest.skip('requires root') + return + + isolation = {'rootfs': option.test_dir} self.load('status_int', isolation=isolation) diff --git a/test/test_settings.py b/test/test_settings.py index b0af6b04..22830a3b 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -146,14 +146,14 @@ Connection: close assert resp['status'] == 200, 'status body read timeout update' - def test_settings_send_timeout(self): + def test_settings_send_timeout(self, temp_dir): self.load('mirror') data_len = 1048576 self.conf({'http': {'send_timeout': 1}}, 'settings') - addr = self.temp_dir + '/sock' + addr = temp_dir + '/sock' self.conf({"unix:" + addr: {'application': 'mirror'}}, 'listeners') @@ -260,3 +260,41 @@ Connection: close assert 'error' in self.conf( {'http': {'max_body_size': -1}}, 'settings' ), 'settings negative value' + + def test_settings_body_buffer_size(self): + self.load('mirror') + + assert 'success' in self.conf( + { + 'http': { + 'max_body_size': 64 * 1024 * 1024, + 'body_buffer_size': 32 * 1024 * 1024, + } + }, + 'settings', + ) + + body = '0123456789abcdef' + resp = self.post(body=body) + assert bool(resp), 'response from application' + assert resp['status'] == 200, 'status' + assert resp['body'] == body, 'body' + + body = '0123456789abcdef' * 1024 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + assert bool(resp), 'response from application 2' + assert resp['status'] == 200, 'status 2' + assert resp['body'] == body, 'body 2' + + body = '0123456789abcdef' * 2 * 1024 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + assert bool(resp), 'response from application 3' + assert resp['status'] == 200, 'status 3' + assert resp['body'] == body, 'body 3' + + body = '0123456789abcdef' * 3 * 1024 * 1024 + resp = self.post(body=body, read_buffer_size=1024 * 1024) + assert bool(resp), 'response from application 4' + assert resp['status'] == 200, 'status 4' + assert resp['body'] == body, 'body 4' + diff --git a/test/test_share_fallback.py b/test/test_share_fallback.py index 391d0836..462da9de 100644 --- a/test/test_share_fallback.py +++ b/test/test_share_fallback.py @@ -1,5 +1,6 @@ import os +from conftest import option from conftest import skip_alert from unit.applications.proto import TestApplicationProto @@ -8,14 +9,12 @@ class TestStatic(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - - os.makedirs(self.temp_dir + '/assets/dir') - with open(self.temp_dir + '/assets/index.html', 'w') as index: + os.makedirs(option.temp_dir + '/assets/dir') + with open(option.temp_dir + '/assets/index.html', 'w') as index: index.write('0123456789') - os.makedirs(self.temp_dir + '/assets/403') - os.chmod(self.temp_dir + '/assets/403', 0o000) + os.makedirs(option.temp_dir + '/assets/403') + os.chmod(option.temp_dir + '/assets/403', 0o000) self._load_conf( { @@ -23,15 +22,13 @@ class TestStatic(TestApplicationProto): "*:7080": {"pass": "routes"}, "*:7081": {"pass": "routes"}, }, - "routes": [{"action": {"share": self.temp_dir + "/assets"}}], + "routes": [{"action": {"share": option.temp_dir + "/assets"}}], "applications": {}, } ) def teardown_method(self): - os.chmod(self.temp_dir + '/assets/403', 0o777) - - super().teardown_method() + os.chmod(option.temp_dir + '/assets/403', 0o777) def action_update(self, conf): assert 'success' in self.conf(conf, 'routes/0/action') @@ -46,9 +43,9 @@ class TestStatic(TestApplicationProto): assert resp['status'] == 200, 'bad path fallback status' assert resp['body'] == '', 'bad path fallback' - def test_fallback_valid_path(self): + def test_fallback_valid_path(self, temp_dir): self.action_update( - {"share": self.temp_dir + "/assets", "fallback": {"return": 200}} + {"share": temp_dir + "/assets", "fallback": {"return": 200}} ) resp = self.get() assert resp['status'] == 200, 'fallback status' @@ -79,11 +76,11 @@ class TestStatic(TestApplicationProto): assert resp['status'] == 200, 'fallback nested status' assert resp['body'] == '', 'fallback nested' - def test_fallback_share(self): + def test_fallback_share(self, temp_dir): self.action_update( { "share": "/blah", - "fallback": {"share": self.temp_dir + "/assets"}, + "fallback": {"share": temp_dir + "/assets"}, } ) diff --git a/test/test_static.py b/test/test_static.py index 0b82b4e8..a65928ca 100644 --- a/test/test_static.py +++ b/test/test_static.py @@ -3,6 +3,7 @@ import socket import pytest +from conftest import option from conftest import waitforfiles from unit.applications.proto import TestApplicationProto @@ -11,13 +12,13 @@ class TestStatic(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - - os.makedirs(self.temp_dir + '/assets/dir') - with open(self.temp_dir + '/assets/index.html', 'w') as index, open( - self.temp_dir + '/assets/README', 'w' - ) as readme, open(self.temp_dir + '/assets/log.log', 'w') as log, open( - self.temp_dir + '/assets/dir/file', 'w' + os.makedirs(option.temp_dir + '/assets/dir') + with open(option.temp_dir + '/assets/index.html', 'w') as index, open( + option.temp_dir + '/assets/README', 'w' + ) as readme, open( + option.temp_dir + '/assets/log.log', 'w' + ) as log, open( + option.temp_dir + '/assets/dir/file', 'w' ) as file: index.write('0123456789') readme.write('readme') @@ -27,7 +28,7 @@ class TestStatic(TestApplicationProto): self._load_conf( { "listeners": {"*:7080": {"pass": "routes"}}, - "routes": [{"action": {"share": self.temp_dir + "/assets"}}], + "routes": [{"action": {"share": option.temp_dir + "/assets"}}], "settings": { "http": { "static": { @@ -54,9 +55,9 @@ class TestStatic(TestApplicationProto): resp['headers']['Content-Type'] == 'text/html' ), 'index not found 2 Content-Type' - def test_static_large_file(self): + def test_static_large_file(self, temp_dir): file_size = 32 * 1024 * 1024 - with open(self.temp_dir + '/assets/large', 'wb') as f: + with open(temp_dir + '/assets/large', 'wb') as f: f.seek(file_size - 1) f.write(b'\0') @@ -65,14 +66,14 @@ class TestStatic(TestApplicationProto): == file_size ), 'large file' - def test_static_etag(self): + def test_static_etag(self, temp_dir): etag = self.get(url='/')['headers']['ETag'] etag_2 = self.get(url='/README')['headers']['ETag'] assert etag != etag_2, 'different ETag' assert etag == self.get(url='/')['headers']['ETag'], 'same ETag' - with open(self.temp_dir + '/assets/index.html', 'w') as f: + with open(temp_dir + '/assets/index.html', 'w') as f: f.write('blah') assert etag != self.get(url='/')['headers']['ETag'], 'new ETag' @@ -83,22 +84,22 @@ class TestStatic(TestApplicationProto): assert resp['headers']['Location'] == '/dir/', 'redirect Location' assert 'Content-Type' not in resp['headers'], 'redirect Content-Type' - def test_static_space_in_name(self): + def test_static_space_in_name(self, temp_dir): os.rename( - self.temp_dir + '/assets/dir/file', - self.temp_dir + '/assets/dir/fi le', + temp_dir + '/assets/dir/file', + temp_dir + '/assets/dir/fi le', ) - assert waitforfiles(self.temp_dir + '/assets/dir/fi le') + assert waitforfiles(temp_dir + '/assets/dir/fi le') assert self.get(url='/dir/fi le')['body'] == 'blah', 'file name' - os.rename(self.temp_dir + '/assets/dir', self.temp_dir + '/assets/di r') - assert waitforfiles(self.temp_dir + '/assets/di r/fi le') + os.rename(temp_dir + '/assets/dir', temp_dir + '/assets/di r') + assert waitforfiles(temp_dir + '/assets/di r/fi le') assert self.get(url='/di r/fi le')['body'] == 'blah', 'dir name' os.rename( - self.temp_dir + '/assets/di r', self.temp_dir + '/assets/ di r ' + temp_dir + '/assets/di r', temp_dir + '/assets/ di r ' ) - assert waitforfiles(self.temp_dir + '/assets/ di r /fi le') + assert waitforfiles(temp_dir + '/assets/ di r /fi le') assert ( self.get(url='/ di r /fi le')['body'] == 'blah' ), 'dir name enclosing' @@ -121,55 +122,58 @@ class TestStatic(TestApplicationProto): ), 'encoded 2' os.rename( - self.temp_dir + '/assets/ di r /fi le', - self.temp_dir + '/assets/ di r / fi le ', + temp_dir + '/assets/ di r /fi le', + temp_dir + '/assets/ di r / fi le ', ) - assert waitforfiles(self.temp_dir + '/assets/ di r / fi le ') + assert waitforfiles(temp_dir + '/assets/ di r / fi le ') assert ( self.get(url='/%20di%20r%20/%20fi%20le%20')['body'] == 'blah' ), 'file name enclosing' try: - open(self.temp_dir + '/ф а', 'a').close() + open(temp_dir + '/ф а', 'a').close() utf8 = True + except KeyboardInterrupt: + raise + except: utf8 = False if utf8: os.rename( - self.temp_dir + '/assets/ di r / fi le ', - self.temp_dir + '/assets/ di r /фа йл', + temp_dir + '/assets/ di r / fi le ', + temp_dir + '/assets/ di r /фа йл', ) - assert waitforfiles(self.temp_dir + '/assets/ di r /фа йл') + assert waitforfiles(temp_dir + '/assets/ di r /фа йл') assert ( self.get(url='/ di r /фа йл')['body'] == 'blah' ), 'file name 2' os.rename( - self.temp_dir + '/assets/ di r ', - self.temp_dir + '/assets/ди ректория', + temp_dir + '/assets/ di r ', + temp_dir + '/assets/ди ректория', ) - assert waitforfiles(self.temp_dir + '/assets/ди ректория/фа йл') + assert waitforfiles(temp_dir + '/assets/ди ректория/фа йл') assert ( self.get(url='/ди ректория/фа йл')['body'] == 'blah' ), 'dir name 2' - def test_static_unix_socket(self): + def test_static_unix_socket(self, temp_dir): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(self.temp_dir + '/assets/unix_socket') + sock.bind(temp_dir + '/assets/unix_socket') assert self.get(url='/unix_socket')['status'] == 404, 'socket' sock.close() - def test_static_unix_fifo(self): - os.mkfifo(self.temp_dir + '/assets/fifo') + def test_static_unix_fifo(self, temp_dir): + os.mkfifo(temp_dir + '/assets/fifo') assert self.get(url='/fifo')['status'] == 404, 'fifo' - def test_static_symlink(self): - os.symlink(self.temp_dir + '/assets/dir', self.temp_dir + '/assets/link') + def test_static_symlink(self, temp_dir): + os.symlink(temp_dir + '/assets/dir', temp_dir + '/assets/link') assert self.get(url='/dir')['status'] == 301, 'dir' assert self.get(url='/dir/file')['status'] == 200, 'file' @@ -312,7 +316,7 @@ class TestStatic(TestApplicationProto): ), 'mime_types same extensions case insensitive' @pytest.mark.skip('not yet') - def test_static_mime_types_invalid(self): + def test_static_mime_types_invalid(self, temp_dir): assert 'error' in self.http( b"""PUT /config/settings/http/static/mime_types/%0%00% HTTP/1.1\r Host: localhost\r @@ -323,5 +327,5 @@ Content-Length: 6\r raw_resp=True, raw=True, sock_type='unix', - addr=self.temp_dir + '/control.unit.sock', + addr=temp_dir + '/control.unit.sock', ), 'mime_types invalid' diff --git a/test/test_tls.py b/test/test_tls.py index 518a834c..4cf8d22c 100644 --- a/test/test_tls.py +++ b/test/test_tls.py @@ -5,6 +5,7 @@ import subprocess import pytest +from conftest import option from conftest import skip_alert from unit.applications.tls import TestApplicationTLS @@ -13,7 +14,7 @@ class TestTLS(TestApplicationTLS): prerequisites = {'modules': {'python': 'any', 'openssl': 'any'}} def findall(self, pattern): - with open(self.temp_dir + '/unit.log', 'r', errors='ignore') as f: + with open(option.temp_dir + '/unit.log', 'r', errors='ignore') as f: return re.findall(pattern, f.read()) def openssl_date_to_sec_epoch(self, date): @@ -134,7 +135,7 @@ class TestTLS(TestApplicationTLS): self.conf_get('/certificates/default/key') == 'RSA (2048 bits)' ), 'certificate key rsa' - def test_tls_certificate_key_ec(self): + def test_tls_certificate_key_ec(self, temp_dir): self.load('empty') self.openssl_conf() @@ -146,7 +147,7 @@ class TestTLS(TestApplicationTLS): '-noout', '-genkey', '-out', - self.temp_dir + '/ec.key', + temp_dir + '/ec.key', '-name', 'prime256v1', ], @@ -162,11 +163,11 @@ class TestTLS(TestApplicationTLS): '-subj', '/CN=ec/', '-config', - self.temp_dir + '/openssl.conf', + temp_dir + '/openssl.conf', '-key', - self.temp_dir + '/ec.key', + temp_dir + '/ec.key', '-out', - self.temp_dir + '/ec.crt', + temp_dir + '/ec.crt', ], stderr=subprocess.STDOUT, ) @@ -208,7 +209,7 @@ class TestTLS(TestApplicationTLS): == 2592000 ), 'certificate validity until' - def test_tls_certificate_chain(self): + def test_tls_certificate_chain(self, temp_dir): self.load('empty') self.certificate('root', False) @@ -221,11 +222,11 @@ class TestTLS(TestApplicationTLS): '-subj', '/CN=int/', '-config', - self.temp_dir + '/openssl.conf', + temp_dir + '/openssl.conf', '-out', - self.temp_dir + '/int.csr', + temp_dir + '/int.csr', '-keyout', - self.temp_dir + '/int.key', + temp_dir + '/int.key', ], stderr=subprocess.STDOUT, ) @@ -238,16 +239,16 @@ class TestTLS(TestApplicationTLS): '-subj', '/CN=end/', '-config', - self.temp_dir + '/openssl.conf', + temp_dir + '/openssl.conf', '-out', - self.temp_dir + '/end.csr', + temp_dir + '/end.csr', '-keyout', - self.temp_dir + '/end.key', + temp_dir + '/end.key', ], stderr=subprocess.STDOUT, ) - with open(self.temp_dir + '/ca.conf', 'w') as f: + with open(temp_dir + '/ca.conf', 'w') as f: f.write( """[ ca ] default_ca = myca @@ -267,16 +268,16 @@ commonName = supplied [ myca_extensions ] basicConstraints = critical,CA:TRUE""" % { - 'dir': self.temp_dir, - 'database': self.temp_dir + '/certindex', - 'certserial': self.temp_dir + '/certserial', + 'dir': temp_dir, + 'database': temp_dir + '/certindex', + 'certserial': temp_dir + '/certserial', } ) - with open(self.temp_dir + '/certserial', 'w') as f: + with open(temp_dir + '/certserial', 'w') as f: f.write('1000') - with open(self.temp_dir + '/certindex', 'w') as f: + with open(temp_dir + '/certindex', 'w') as f: f.write('') subprocess.call( @@ -287,15 +288,15 @@ basicConstraints = critical,CA:TRUE""" '-subj', '/CN=int/', '-config', - self.temp_dir + '/ca.conf', + temp_dir + '/ca.conf', '-keyfile', - self.temp_dir + '/root.key', + temp_dir + '/root.key', '-cert', - self.temp_dir + '/root.crt', + temp_dir + '/root.crt', '-in', - self.temp_dir + '/int.csr', + temp_dir + '/int.csr', '-out', - self.temp_dir + '/int.crt', + temp_dir + '/int.crt', ], stderr=subprocess.STDOUT, ) @@ -308,22 +309,22 @@ basicConstraints = critical,CA:TRUE""" '-subj', '/CN=end/', '-config', - self.temp_dir + '/ca.conf', + temp_dir + '/ca.conf', '-keyfile', - self.temp_dir + '/int.key', + temp_dir + '/int.key', '-cert', - self.temp_dir + '/int.crt', + temp_dir + '/int.crt', '-in', - self.temp_dir + '/end.csr', + temp_dir + '/end.csr', '-out', - self.temp_dir + '/end.crt', + temp_dir + '/end.crt', ], stderr=subprocess.STDOUT, ) - crt_path = self.temp_dir + '/end-int.crt' - end_path = self.temp_dir + '/end.crt' - int_path = self.temp_dir + '/int.crt' + crt_path = temp_dir + '/end-int.crt' + end_path = temp_dir + '/end.crt' + int_path = temp_dir + '/int.crt' with open(crt_path, 'wb') as crt, open(end_path, 'rb') as end, open( int_path, 'rb' @@ -333,7 +334,7 @@ basicConstraints = critical,CA:TRUE""" self.context = ssl.create_default_context() self.context.check_hostname = False self.context.verify_mode = ssl.CERT_REQUIRED - self.context.load_verify_locations(self.temp_dir + '/root.crt') + self.context.load_verify_locations(temp_dir + '/root.crt') # incomplete chain @@ -485,6 +486,10 @@ basicConstraints = critical,CA:TRUE""" resp = self.get_ssl( headers={'Host': 'localhost', 'Connection': 'close'}, sock=sock ) + + except KeyboardInterrupt: + raise + except: resp = None diff --git a/test/test_upstreams_rr.py b/test/test_upstreams_rr.py index 2ecf1d9a..c20d6054 100644 --- a/test/test_upstreams_rr.py +++ b/test/test_upstreams_rr.py @@ -9,8 +9,6 @@ class TestUpstreamsRR(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} def setup_method(self): - super().setup_method() - assert 'success' in self.conf( { "listeners": { @@ -391,9 +389,9 @@ Connection: close assert sum(resps) == 100, 'post sum' assert abs(resps[0] - resps[1]) <= self.cpu_count, 'post' - def test_upstreams_rr_unix(self): - addr_0 = self.temp_dir + '/sock_0' - addr_1 = self.temp_dir + '/sock_1' + def test_upstreams_rr_unix(self, temp_dir): + addr_0 = temp_dir + '/sock_0' + addr_1 = temp_dir + '/sock_1' assert 'success' in self.conf( { diff --git a/test/test_usr1.py b/test/test_usr1.py index 2e48c18f..3e44e4c5 100644 --- a/test/test_usr1.py +++ b/test/test_usr1.py @@ -1,6 +1,7 @@ import os from subprocess import call +from conftest import unit_stop from conftest import waitforfiles from unit.applications.lang.python import TestApplicationPython @@ -8,12 +9,12 @@ from unit.applications.lang.python import TestApplicationPython class TestUSR1(TestApplicationPython): prerequisites = {'modules': {'python': 'any'}} - def test_usr1_access_log(self): + def test_usr1_access_log(self, temp_dir): self.load('empty') log = 'access.log' log_new = 'new.log' - log_path = self.temp_dir + '/' + log + log_path = temp_dir + '/' + log assert 'success' in self.conf( '"' + log_path + '"', 'access_log' @@ -21,7 +22,7 @@ class TestUSR1(TestApplicationPython): assert waitforfiles(log_path), 'open' - os.rename(log_path, self.temp_dir + '/' + log_new) + os.rename(log_path, temp_dir + '/' + log_new) assert self.get()['status'] == 200 @@ -31,7 +32,7 @@ class TestUSR1(TestApplicationPython): ), 'rename new' assert not os.path.isfile(log_path), 'rename old' - with open(self.temp_dir + '/unit.pid', 'r') as f: + with open(temp_dir + '/unit.pid', 'r') as f: pid = f.read().rstrip() call(['kill', '-s', 'USR1', pid]) @@ -40,7 +41,7 @@ class TestUSR1(TestApplicationPython): assert self.get(url='/usr1')['status'] == 200 - self.stop() + unit_stop() assert ( self.wait_for_record(r'"GET /usr1 HTTP/1.1" 200 0 "-" "-"', log) @@ -48,12 +49,12 @@ class TestUSR1(TestApplicationPython): ), 'reopen 2' assert self.search_in_log(r'/usr1', log_new) is None, 'rename new 2' - def test_usr1_unit_log(self): + def test_usr1_unit_log(self, temp_dir): self.load('log_body') log_new = 'new.log' - log_path = self.temp_dir + '/unit.log' - log_path_new = self.temp_dir + '/' + log_new + log_path = temp_dir + '/unit.log' + log_path_new = temp_dir + '/' + log_new os.rename(log_path, log_path_new) @@ -63,7 +64,7 @@ class TestUSR1(TestApplicationPython): assert self.wait_for_record(body, log_new) is not None, 'rename new' assert not os.path.isfile(log_path), 'rename old' - with open(self.temp_dir + '/unit.pid', 'r') as f: + with open(temp_dir + '/unit.pid', 'r') as f: pid = f.read().rstrip() call(['kill', '-s', 'USR1', pid]) @@ -73,7 +74,7 @@ class TestUSR1(TestApplicationPython): body = 'body_for_a_log_unit' assert self.post(body=body)['status'] == 200 - self.stop() + unit_stop() assert self.wait_for_record(body) is not None, 'rename new' assert self.search_in_log(body, log_new) is None, 'rename new 2' diff --git a/test/test_variables.py b/test/test_variables.py index c458b636..bbb8f769 100644 --- a/test/test_variables.py +++ b/test/test_variables.py @@ -5,8 +5,6 @@ class TestVariables(TestApplicationProto): prerequisites = {} def setup_method(self): - super().setup_method() - assert 'success' in self.conf( { "listeners": {"*:7080": {"pass": "routes/$method"}}, diff --git a/test/unit/applications/lang/go.py b/test/unit/applications/lang/go.py index 7715bd6c..866dec47 100644 --- a/test/unit/applications/lang/go.py +++ b/test/unit/applications/lang/go.py @@ -7,8 +7,8 @@ from unit.applications.proto import TestApplicationProto class TestApplicationGo(TestApplicationProto): def prepare_env(self, script, name, static=False): - if not os.path.exists(self.temp_dir + '/go'): - os.mkdir(self.temp_dir + '/go') + if not os.path.exists(option.temp_dir + '/go'): + os.mkdir(option.temp_dir + '/go') env = os.environ.copy() env['GOPATH'] = option.current_dir + '/build/go' @@ -22,7 +22,7 @@ class TestApplicationGo(TestApplicationProto): '-ldflags', '-extldflags "-static"', '-o', - self.temp_dir + '/go/' + name, + option.temp_dir + '/go/' + name, option.test_dir + '/go/' + script + '/' + name + '.go', ] else: @@ -30,14 +30,20 @@ class TestApplicationGo(TestApplicationProto): 'go', 'build', '-o', - self.temp_dir + '/go/' + name, + option.temp_dir + '/go/' + name, option.test_dir + '/go/' + script + '/' + name + '.go', ] + if option.detailed: + print("\n$ GOPATH=" + env['GOPATH'] + " " + " ".join(args)) + try: process = subprocess.Popen(args, env=env) process.communicate() + except KeyboardInterrupt: + raise + except: return None @@ -47,7 +53,7 @@ class TestApplicationGo(TestApplicationProto): static_build = False wdir = option.test_dir + "/go/" + script - executable = self.temp_dir + "/go/" + name + executable = option.temp_dir + "/go/" + name if 'isolation' in kwargs and 'rootfs' in kwargs['isolation']: wdir = "/go/" diff --git a/test/unit/applications/lang/java.py b/test/unit/applications/lang/java.py index 01cbfa0b..0ff85187 100644 --- a/test/unit/applications/lang/java.py +++ b/test/unit/applications/lang/java.py @@ -9,8 +9,10 @@ from unit.applications.proto import TestApplicationProto class TestApplicationJava(TestApplicationProto): - def load(self, script, name='app', **kwargs): - app_path = self.temp_dir + '/java' + application_type = "java" + + def prepare_env(self, script): + app_path = option.temp_dir + '/java' web_inf_path = app_path + '/WEB-INF/' classes_path = web_inf_path + 'classes/' script_path = option.test_dir + '/java/' + script + '/' @@ -50,7 +52,7 @@ class TestApplicationJava(TestApplicationProto): os.makedirs(classes_path) classpath = ( - option.current_dir + '/build/tomcat-servlet-api-9.0.13.jar' + option.current_dir + '/build/tomcat-servlet-api-9.0.39.jar' ) ws_jars = glob.glob( @@ -62,18 +64,28 @@ class TestApplicationJava(TestApplicationProto): javac = [ 'javac', + '-target', '8', '-source', '8', '-nowarn', '-encoding', 'utf-8', '-d', classes_path, '-classpath', classpath + ':' + ws_jars[0], ] javac.extend(src) + if option.detailed: + print("\n$ " + " ".join(javac)) + try: process = subprocess.Popen(javac, stderr=subprocess.STDOUT) process.communicate() + except KeyboardInterrupt: + raise + except: - pytest.fail('Cann\'t run javac process.') + pytest.fail('Can\'t run javac process.') + + def load(self, script, **kwargs): + self.prepare_env(script) self._load_conf( { @@ -81,10 +93,13 @@ class TestApplicationJava(TestApplicationProto): "applications": { script: { "unit_jars": option.current_dir + '/build', - "type": 'java', + "type": self.get_application_type(), "processes": {"spare": 0}, - "working_directory": script_path, - "webapp": app_path, + "working_directory": option.test_dir + + '/java/' + + script + + '/', + "webapp": option.temp_dir + '/java', } }, }, diff --git a/test/unit/applications/lang/node.py b/test/unit/applications/lang/node.py index 877fc461..98fd9ffc 100644 --- a/test/unit/applications/lang/node.py +++ b/test/unit/applications/lang/node.py @@ -7,21 +7,24 @@ from unit.applications.proto import TestApplicationProto class TestApplicationNode(TestApplicationProto): - def load(self, script, name='app.js', **kwargs): + def prepare_env(self, script): # copy application shutil.copytree( - option.test_dir + '/node/' + script, self.temp_dir + '/node' + option.test_dir + '/node/' + script, option.temp_dir + '/node' ) # copy modules shutil.copytree( option.current_dir + '/node/node_modules', - self.temp_dir + '/node/node_modules', + option.temp_dir + '/node/node_modules', ) - public_dir(self.temp_dir + '/node') + public_dir(option.temp_dir + '/node') + + def load(self, script, name='app.js', **kwargs): + self.prepare_env(script) self._load_conf( { @@ -32,7 +35,7 @@ class TestApplicationNode(TestApplicationProto): script: { "type": "external", "processes": {"spare": 0}, - "working_directory": self.temp_dir + '/node', + "working_directory": option.temp_dir + '/node', "executable": name, } }, diff --git a/test/unit/applications/lang/perl.py b/test/unit/applications/lang/perl.py index a27c7649..9dc24ace 100644 --- a/test/unit/applications/lang/perl.py +++ b/test/unit/applications/lang/perl.py @@ -7,17 +7,13 @@ class TestApplicationPerl(TestApplicationProto): def load(self, script, name='psgi.pl', **kwargs): script_path = option.test_dir + '/perl/' + script - appication_type = self.get_appication_type() - - if appication_type is None: - appication_type = self.application_type self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, "applications": { script: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "working_directory": script_path, "script": script_path + '/' + name, diff --git a/test/unit/applications/lang/php.py b/test/unit/applications/lang/php.py index 2d50df2e..3dbb32f5 100644 --- a/test/unit/applications/lang/php.py +++ b/test/unit/applications/lang/php.py @@ -1,4 +1,7 @@ from conftest import option +import os +import shutil + from unit.applications.proto import TestApplicationProto @@ -7,17 +10,24 @@ class TestApplicationPHP(TestApplicationProto): def load(self, script, index='index.php', **kwargs): script_path = option.test_dir + '/php/' + script - appication_type = self.get_appication_type() - if appication_type is None: - appication_type = self.application_type + if kwargs.get('isolation') and kwargs['isolation'].get('rootfs'): + rootfs = kwargs['isolation']['rootfs'] + + if not os.path.exists(rootfs + '/app/php/'): + os.makedirs(rootfs + '/app/php/') + + if not os.path.exists(rootfs + '/app/php/' + script): + shutil.copytree(script_path, rootfs + '/app/php/' + script) + + script_path = '/app/php/' + script self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, "applications": { script: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "root": script_path, "working_directory": script_path, diff --git a/test/unit/applications/lang/python.py b/test/unit/applications/lang/python.py index 47b95dac..792a86fa 100644 --- a/test/unit/applications/lang/python.py +++ b/test/unit/applications/lang/python.py @@ -12,7 +12,6 @@ class TestApplicationPython(TestApplicationProto): load_module = "wsgi" def load(self, script, name=None, module=None, **kwargs): - print() if name is None: name = script @@ -35,25 +34,25 @@ class TestApplicationPython(TestApplicationProto): script_path = '/app/python/' + name - appication_type = self.get_appication_type() + app = { + "type": self.get_application_type(), + "processes": kwargs.pop('processes', {"spare": 0}), + "path": script_path, + "working_directory": script_path, + "module": module, + } - if appication_type is None: - appication_type = self.application_type + for attr in ('callable', 'home', 'limits', 'path', 'protocol', + 'threads'): + if attr in kwargs: + app[attr] = kwargs.pop(attr) self._load_conf( { "listeners": { "*:7080": {"pass": "applications/" + quote(name, '')} }, - "applications": { - name: { - "type": appication_type, - "processes": {"spare": 0}, - "path": script_path, - "working_directory": script_path, - "module": module, - } - }, + "applications": {name: app}, }, **kwargs ) diff --git a/test/unit/applications/lang/ruby.py b/test/unit/applications/lang/ruby.py index bc3cefc6..82d66e65 100644 --- a/test/unit/applications/lang/ruby.py +++ b/test/unit/applications/lang/ruby.py @@ -7,17 +7,13 @@ class TestApplicationRuby(TestApplicationProto): def load(self, script, name='config.ru', **kwargs): script_path = option.test_dir + '/ruby/' + script - appication_type = self.get_appication_type() - - if appication_type is None: - appication_type = self.application_type self._load_conf( { "listeners": {"*:7080": {"pass": "applications/" + script}}, "applications": { script: { - "type": appication_type, + "type": self.get_application_type(), "processes": {"spare": 0}, "working_directory": script_path, "script": script_path + '/' + name, diff --git a/test/unit/applications/proto.py b/test/unit/applications/proto.py index 2f748c21..6e760c70 100644 --- a/test/unit/applications/proto.py +++ b/test/unit/applications/proto.py @@ -7,6 +7,8 @@ from unit.control import TestControl class TestApplicationProto(TestControl): + application_type = None + def sec_epoch(self): return time.mktime(time.gmtime()) @@ -14,7 +16,7 @@ class TestApplicationProto(TestControl): return time.mktime(time.strptime(date, template)) def search_in_log(self, pattern, name='unit.log'): - with open(self.temp_dir + '/' + name, 'r', errors='ignore') as f: + with open(option.temp_dir + '/' + name, 'r', errors='ignore') as f: return re.search(pattern, f.read()) def wait_for_record(self, pattern, name='unit.log'): @@ -28,15 +30,12 @@ class TestApplicationProto(TestControl): return found - def get_appication_type(self): + def get_application_type(self): current_test = ( os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0] ) - if current_test in option.generated_tests: - return option.generated_tests[current_test] - - return None + return option.generated_tests.get(current_test, self.application_type) def _load_conf(self, conf, **kwargs): if 'applications' in conf: diff --git a/test/unit/applications/tls.py b/test/unit/applications/tls.py index fdf681ae..fb1b112c 100644 --- a/test/unit/applications/tls.py +++ b/test/unit/applications/tls.py @@ -8,8 +8,6 @@ from unit.applications.proto import TestApplicationProto class TestApplicationTLS(TestApplicationProto): def setup_method(self): - super().setup_method() - self.context = ssl.create_default_context() self.context.check_hostname = False self.context.verify_mode = ssl.CERT_NONE @@ -24,9 +22,9 @@ class TestApplicationTLS(TestApplicationProto): '-x509', '-new', '-subj', '/CN=' + name + '/', - '-config', self.temp_dir + '/openssl.conf', - '-out', self.temp_dir + '/' + name + '.crt', - '-keyout', self.temp_dir + '/' + name + '.key', + '-config', option.temp_dir + '/openssl.conf', + '-out', option.temp_dir + '/' + name + '.crt', + '-keyout', option.temp_dir + '/' + name + '.key', ], stderr=subprocess.STDOUT, ) @@ -38,8 +36,8 @@ class TestApplicationTLS(TestApplicationProto): if key is None: key = crt - key_path = self.temp_dir + '/' + key + '.key' - crt_path = self.temp_dir + '/' + crt + '.crt' + key_path = option.temp_dir + '/' + key + '.key' + crt_path = option.temp_dir + '/' + crt + '.crt' with open(key_path, 'rb') as k, open(crt_path, 'rb') as c: return self.conf(k.read() + c.read(), '/certificates/' + crt) @@ -66,7 +64,7 @@ class TestApplicationTLS(TestApplicationProto): return ssl.get_server_certificate(addr, ssl_version=ssl_version) def openssl_conf(self): - conf_path = self.temp_dir + '/openssl.conf' + conf_path = option.temp_dir + '/openssl.conf' if os.path.exists(conf_path): return diff --git a/test/unit/check/go.py b/test/unit/check/go.py index dd2150eb..35b0c2d5 100644 --- a/test/unit/check/go.py +++ b/test/unit/check/go.py @@ -25,5 +25,8 @@ def check_go(current_dir, temp_dir, test_dir): if process.returncode == 0: return True + except KeyboardInterrupt: + raise + except: return None diff --git a/test/unit/control.py b/test/unit/control.py index 6fd350f4..f05aa827 100644 --- a/test/unit/control.py +++ b/test/unit/control.py @@ -1,5 +1,6 @@ import json +from conftest import option from unit.http import TestHTTP @@ -53,7 +54,7 @@ class TestControl(TestHTTP): args = { 'url': url, 'sock_type': 'unix', - 'addr': self.temp_dir + '/control.unit.sock', + 'addr': option.temp_dir + '/control.unit.sock', } if conf is not None: diff --git a/test/unit/feature/isolation.py b/test/unit/feature/isolation.py index c6f6f3c0..7877c03a 100644 --- a/test/unit/feature/isolation.py +++ b/test/unit/feature/isolation.py @@ -3,11 +3,8 @@ import os from unit.applications.lang.go import TestApplicationGo from unit.applications.lang.java import TestApplicationJava from unit.applications.lang.node import TestApplicationNode -from unit.applications.lang.perl import TestApplicationPerl -from unit.applications.lang.php import TestApplicationPHP -from unit.applications.lang.python import TestApplicationPython -from unit.applications.lang.ruby import TestApplicationRuby from unit.applications.proto import TestApplicationProto +from conftest import option class TestFeatureIsolation(TestApplicationProto): @@ -16,40 +13,119 @@ class TestFeatureIsolation(TestApplicationProto): def check(self, available, temp_dir): test_conf = {"namespaces": {"credential": True}} - module = '' - app = 'empty' + conf = '' if 'go' in available['modules']: - module = TestApplicationGo() + TestApplicationGo().prepare_env('empty', 'app') + + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/go/empty", + "executable": option.temp_dir + "/go/app", + "isolation": {"namespaces": {"credential": True}}, + }, + }, + } - elif 'java' in available['modules']: - module = TestApplicationJava() - - elif 'node' in available['modules']: - module = TestApplicationNode() - app = 'basic' - - elif 'perl' in available['modules']: - module = TestApplicationPerl() - app = 'body_empty' + elif 'python' in available['modules']: + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "python", + "processes": {"spare": 0}, + "path": option.test_dir + "/python/empty", + "working_directory": option.test_dir + "/python/empty", + "module": "wsgi", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'php' in available['modules']: - module = TestApplicationPHP() - app = 'phpinfo' - - elif 'python' in available['modules']: - module = TestApplicationPython() + conf = { + "listeners": {"*:7080": {"pass": "applications/phpinfo"}}, + "applications": { + "phpinfo": { + "type": "php", + "processes": {"spare": 0}, + "root": option.test_dir + "/php/phpinfo", + "working_directory": option.test_dir + "/php/phpinfo", + "index": "index.php", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } elif 'ruby' in available['modules']: - module = TestApplicationRuby() + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "type": "ruby", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/ruby/empty", + "script": option.test_dir + "/ruby/empty/config.ru", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } - if not module: - return + elif 'java' in available['modules']: + TestApplicationJava().prepare_env('empty') + + conf = { + "listeners": {"*:7080": {"pass": "applications/empty"}}, + "applications": { + "empty": { + "unit_jars": option.current_dir + "/build", + "type": "java", + "processes": {"spare": 0}, + "working_directory": option.test_dir + "/java/empty/", + "webapp": option.temp_dir + "/java", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } - module.temp_dir = temp_dir - module.load(app) + elif 'node' in available['modules']: + TestApplicationNode().prepare_env('basic') + + conf = { + "listeners": {"*:7080": {"pass": "applications/basic"}}, + "applications": { + "basic": { + "type": "external", + "processes": {"spare": 0}, + "working_directory": option.temp_dir + "/node", + "executable": "app.js", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + elif 'perl' in available['modules']: + conf = { + "listeners": {"*:7080": {"pass": "applications/body_empty"}}, + "applications": { + "body_empty": { + "type": "perl", + "processes": {"spare": 0}, + "working_directory": option.test_dir + + "/perl/body_empty", + "script": option.test_dir + "/perl/body_empty/psgi.pl", + "isolation": {"namespaces": {"credential": True}}, + } + }, + } + + else: + return - resp = module.conf(test_conf, 'applications/' + app + '/isolation') - if 'success' not in resp: + if 'success' not in self.conf(conf): return userns = self.getns('user') diff --git a/test/unit/http.py b/test/unit/http.py index 7845f9a8..8d964978 100644 --- a/test/unit/http.py +++ b/test/unit/http.py @@ -5,7 +5,6 @@ import os import re import select import socket -import time import pytest from conftest import option @@ -188,6 +187,10 @@ class TestHTTP(TestUnit): try: part = sock.recv(buff_size) + + except KeyboardInterrupt: + raise + except: break @@ -243,7 +246,8 @@ class TestHTTP(TestUnit): try: last_size = int(chunks[-2], 16) - except: + + except ValueError: pytest.fail('Invalid zero size chunk') if last_size != 0 or chunks[-1] != b'': @@ -253,7 +257,8 @@ class TestHTTP(TestUnit): while len(chunks) >= 2: try: size = int(chunks.pop(0), 16) - except: + + except ValueError: pytest.fail('Invalid chunk size %s' % str(size)) if size == 0: @@ -283,23 +288,6 @@ class TestHTTP(TestUnit): def getjson(self, **kwargs): return self.get(json=True, **kwargs) - def waitforsocket(self, port): - ret = False - - for i in range(50): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('127.0.0.1', port)) - ret = True - break - except: - sock.close() - time.sleep(0.1) - - sock.close() - - assert ret, 'socket connected' - def form_encode(self, fields): is_multipart = False diff --git a/test/unit/main.py b/test/unit/main.py index d5940995..488b3f4d 100644 --- a/test/unit/main.py +++ b/test/unit/main.py @@ -1,55 +1,19 @@ -import atexit -import os -import re -import shutil -import signal -import stat -import subprocess -import tempfile -import time -from multiprocessing import Process - import pytest -from conftest import _check_alerts -from conftest import _print_log from conftest import option -from conftest import public_dir -from conftest import waitforfiles class TestUnit(): @classmethod def setup_class(cls, complete_check=True): - cls.available = option.available - unit = TestUnit() - - unit._run() - - # read unit.log - - for i in range(50): - with open(unit.temp_dir + '/unit.log', 'r') as f: - log = f.read() - m = re.search('controller started', log) - - if m is None: - time.sleep(0.1) - else: - break - - if m is None: - _print_log(path=unit.temp_dir + '/unit.log') - exit("Unit is writing log too long") - - def check(available, prerequisites): + def check(): missed = [] # check modules - if 'modules' in prerequisites: - available_modules = list(available['modules'].keys()) + if 'modules' in cls.prerequisites: + available_modules = list(option.available['modules'].keys()) - for module in prerequisites['modules']: + for module in cls.prerequisites['modules']: if module in available_modules: continue @@ -60,10 +24,10 @@ class TestUnit(): # check features - if 'features' in prerequisites: - available_features = list(available['features'].keys()) + if 'features' in cls.prerequisites: + available_features = list(option.available['features'].keys()) - for feature in prerequisites['features']: + for feature in cls.prerequisites['features']: if feature in available_features: continue @@ -72,132 +36,7 @@ class TestUnit(): if missed: pytest.skip(', '.join(missed) + ' feature(s) not supported') - def destroy(): - unit.stop() - _check_alerts(log) - shutil.rmtree(unit.temp_dir) - - def complete(): - destroy() - check(cls.available, cls.prerequisites) - if complete_check: - complete() - else: - unit.complete = complete - return unit - - def setup_method(self): - self._run() - - def _run(self): - build_dir = option.current_dir + '/build' - self.unitd = build_dir + '/unitd' - - if not os.path.isfile(self.unitd): - exit("Could not find unit") - - self.temp_dir = tempfile.mkdtemp(prefix='unit-test-') - - public_dir(self.temp_dir) - - if oct(stat.S_IMODE(os.stat(build_dir).st_mode)) != '0o777': - public_dir(build_dir) - - os.mkdir(self.temp_dir + '/state') - - with open(self.temp_dir + '/unit.log', 'w') as log: - self._p = subprocess.Popen( - [ - self.unitd, - '--no-daemon', - '--modules', build_dir, - '--state', self.temp_dir + '/state', - '--pid', self.temp_dir + '/unit.pid', - '--log', self.temp_dir + '/unit.log', - '--control', 'unix:' + self.temp_dir + '/control.unit.sock', - '--tmp', self.temp_dir, - ], - stderr=log, - ) - - atexit.register(self.stop) - - if not waitforfiles(self.temp_dir + '/control.unit.sock'): - _print_log(path=self.temp_dir + '/unit.log') - exit("Could not start unit") - - self._started = True - - def teardown_method(self): - self.stop() - - # check unit.log for alerts - - unit_log = self.temp_dir + '/unit.log' - - with open(unit_log, 'r', encoding='utf-8', errors='ignore') as f: - _check_alerts(f.read()) - - # remove unit.log - - if not option.save_log: - shutil.rmtree(self.temp_dir) + check() else: - _print_log(path=self.temp_dir) - - assert self.stop_errors == [None, None], 'stop errors' - - def stop(self): - if not self._started: - return - - self.stop_errors = [] - - self.stop_errors.append(self._stop()) - - self.stop_errors.append(self.stop_processes()) - - atexit.unregister(self.stop) - - self._started = False - - def _stop(self): - if self._p.poll() is not None: - return - - with self._p as p: - p.send_signal(signal.SIGQUIT) - - try: - retcode = p.wait(15) - if retcode: - return 'Child process terminated with code ' + str(retcode) - except: - p.kill() - return 'Could not terminate unit' - - def run_process(self, target, *args): - if not hasattr(self, '_processes'): - self._processes = [] - - process = Process(target=target, args=args) - process.start() - - self._processes.append(process) - - def stop_processes(self): - if not hasattr(self, '_processes'): - return - - fail = False - for process in self._processes: - if process.is_alive(): - process.terminate() - process.join(timeout=15) - - if process.is_alive(): - fail = True - - if fail: - return 'Fail to stop process' + return check @@ -1,5 +1,5 @@ # Copyright (C) NGINX, Inc. -NXT_VERSION=1.20.0 -NXT_VERNUM=12000 +NXT_VERSION=1.21.0 +NXT_VERNUM=12100 |