From d94dac091f6a6878f10cfc8fa1ef059dd6bfe964 Mon Sep 17 00:00:00 2001 From: Max Romanov Date: Mon, 14 Sep 2020 13:27:02 +0300 Subject: Python: split module initialization from WSGI implementation. This is required for futher ASGI implementation. --- src/python/nxt_python.c | 331 +++++++++++++++++++++++++++++++++++++++++++ src/python/nxt_python.h | 31 ++++ src/python/nxt_python_wsgi.c | 310 ++++------------------------------------ 3 files changed, 387 insertions(+), 285 deletions(-) create mode 100644 src/python/nxt_python.c create mode 100644 src/python/nxt_python.h (limited to 'src/python') diff --git a/src/python/nxt_python.c b/src/python/nxt_python.c new file mode 100644 index 00000000..5b6021bb --- /dev/null +++ b/src/python/nxt_python.c @@ -0,0 +1,331 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + + +#include + +#include +#include +#include + +#include + +#include NXT_PYTHON_MOUNTS_H + + +#if PY_MAJOR_VERSION == 3 +#define PyString_FromStringAndSize(str, size) \ + PyUnicode_DecodeLatin1((str), (size), "strict") + +#else +#define PyUnicode_InternInPlace PyString_InternInPlace +#endif + +static nxt_int_t nxt_python_start(nxt_task_t *task, + nxt_process_data_t *data); +static void nxt_python_atexit(void); + +static uint32_t compat[] = { + NXT_VERNUM, NXT_DEBUG, +}; + + +NXT_EXPORT nxt_app_module_t nxt_app_module = { + sizeof(compat), + compat, + nxt_string("python"), + PY_VERSION, + nxt_python_mounts, + nxt_nitems(nxt_python_mounts), + NULL, + nxt_python_start, +}; + +static PyObject *nxt_py_stderr_flush; +PyObject *nxt_py_application; + +#if PY_MAJOR_VERSION == 3 +static wchar_t *nxt_py_home; +#else +static char *nxt_py_home; +#endif + + +static nxt_int_t +nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) +{ + int rc; + char *nxt_py_module; + size_t len; + PyObject *obj, *pypath, *module; + nxt_unit_ctx_t *unit_ctx; + nxt_unit_init_t python_init; + nxt_common_app_conf_t *app_conf; + nxt_python_app_conf_t *c; +#if PY_MAJOR_VERSION == 3 + char *path; + size_t size; + nxt_int_t pep405; + + static const char pyvenv[] = "/pyvenv.cfg"; + static const char bin_python[] = "/bin/python"; +#endif + + app_conf = data->app; + c = &app_conf->u.python; + + if (c->home != NULL) { + len = nxt_strlen(c->home); + +#if PY_MAJOR_VERSION == 3 + + path = nxt_malloc(len + sizeof(pyvenv)); + if (nxt_slow_path(path == NULL)) { + nxt_alert(task, "Failed to allocate memory"); + return NXT_ERROR; + } + + nxt_memcpy(path, c->home, len); + nxt_memcpy(path + len, pyvenv, sizeof(pyvenv)); + + pep405 = (access(path, R_OK) == 0); + + nxt_free(path); + + if (pep405) { + size = (len + sizeof(bin_python)) * sizeof(wchar_t); + + } else { + size = (len + 1) * sizeof(wchar_t); + } + + nxt_py_home = nxt_malloc(size); + if (nxt_slow_path(nxt_py_home == NULL)) { + nxt_alert(task, "Failed to allocate memory"); + return NXT_ERROR; + } + + if (pep405) { + mbstowcs(nxt_py_home, c->home, len); + mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python)); + Py_SetProgramName(nxt_py_home); + + } else { + mbstowcs(nxt_py_home, c->home, len + 1); + Py_SetPythonHome(nxt_py_home); + } + +#else + nxt_py_home = nxt_malloc(len + 1); + if (nxt_slow_path(nxt_py_home == NULL)) { + nxt_alert(task, "Failed to allocate memory"); + return NXT_ERROR; + } + + nxt_memcpy(nxt_py_home, c->home, len + 1); + Py_SetPythonHome(nxt_py_home); +#endif + } + + Py_InitializeEx(0); + + module = NULL; + obj = NULL; + + obj = PySys_GetObject((char *) "stderr"); + if (nxt_slow_path(obj == NULL)) { + nxt_alert(task, "Python failed to get \"sys.stderr\" object"); + goto fail; + } + + nxt_py_stderr_flush = PyObject_GetAttrString(obj, "flush"); + if (nxt_slow_path(nxt_py_stderr_flush == NULL)) { + nxt_alert(task, "Python failed to get \"flush\" attribute of " + "\"sys.stderr\" object"); + goto fail; + } + + /* obj is a Borrowed reference. */ + + if (c->path.length > 0) { + obj = PyString_FromStringAndSize((char *) c->path.start, + c->path.length); + + if (nxt_slow_path(obj == NULL)) { + nxt_alert(task, "Python failed to create string object \"%V\"", + &c->path); + goto fail; + } + + pypath = PySys_GetObject((char *) "path"); + + if (nxt_slow_path(pypath == NULL)) { + nxt_alert(task, "Python failed to get \"sys.path\" list"); + goto fail; + } + + if (nxt_slow_path(PyList_Insert(pypath, 0, obj) != 0)) { + nxt_alert(task, "Python failed to insert \"%V\" into \"sys.path\"", + &c->path); + goto fail; + } + + Py_DECREF(obj); + } + + obj = Py_BuildValue("[s]", "unit"); + if (nxt_slow_path(obj == NULL)) { + nxt_alert(task, "Python failed to create the \"sys.argv\" list"); + goto fail; + } + + if (nxt_slow_path(PySys_SetObject((char *) "argv", obj) != 0)) { + nxt_alert(task, "Python failed to set the \"sys.argv\" list"); + goto fail; + } + + Py_CLEAR(obj); + + nxt_py_module = nxt_alloca(c->module.length + 1); + nxt_memcpy(nxt_py_module, c->module.start, c->module.length); + nxt_py_module[c->module.length] = '\0'; + + module = PyImport_ImportModule(nxt_py_module); + if (nxt_slow_path(module == NULL)) { + nxt_alert(task, "Python failed to import module \"%s\"", nxt_py_module); + nxt_python_print_exception(); + goto fail; + } + + obj = PyDict_GetItemString(PyModule_GetDict(module), "application"); + if (nxt_slow_path(obj == NULL)) { + nxt_alert(task, "Python failed to get \"application\" " + "from module \"%s\"", nxt_py_module); + goto fail; + } + + if (nxt_slow_path(PyCallable_Check(obj) == 0)) { + nxt_alert(task, "\"application\" in module \"%s\" " + "is not a callable object", nxt_py_module); + goto fail; + } + + nxt_py_application = obj; + obj = NULL; + + Py_INCREF(nxt_py_application); + + Py_CLEAR(module); + + nxt_unit_default_init(task, &python_init); + + python_init.shm_limit = data->app->shm_limit; + + rc = nxt_python_wsgi_init(task, &python_init); + if (nxt_slow_path(rc == NXT_ERROR)) { + goto fail; + } + + unit_ctx = nxt_unit_init(&python_init); + if (nxt_slow_path(unit_ctx == NULL)) { + goto fail; + } + + rc = nxt_python_wsgi_run(unit_ctx); + + nxt_unit_done(unit_ctx); + + nxt_python_atexit(); + + exit(rc); + + return NXT_OK; + +fail: + + Py_XDECREF(obj); + Py_XDECREF(module); + + nxt_python_atexit(); + + return NXT_ERROR; +} + + +nxt_int_t +nxt_python_init_strings(nxt_python_string_t *pstr) +{ + PyObject *obj; + + while (pstr->string.start != NULL) { + obj = PyString_FromStringAndSize((char *) pstr->string.start, + pstr->string.length); + if (nxt_slow_path(obj == NULL)) { + return NXT_ERROR; + } + + PyUnicode_InternInPlace(&obj); + + *pstr->object_p = obj; + + pstr++; + } + + return NXT_OK; +} + + +void +nxt_python_done_strings(nxt_python_string_t *pstr) +{ + PyObject *obj; + + while (pstr->string.start != NULL) { + obj = *pstr->object_p; + + Py_XDECREF(obj); + *pstr->object_p = NULL; + + pstr++; + } +} + + +static void +nxt_python_atexit(void) +{ + nxt_python_wsgi_done(); + + Py_XDECREF(nxt_py_stderr_flush); + Py_XDECREF(nxt_py_application); + + Py_Finalize(); + + if (nxt_py_home != NULL) { + nxt_free(nxt_py_home); + } +} + + +void +nxt_python_print_exception(void) +{ + PyErr_Print(); + +#if PY_MAJOR_VERSION == 3 + /* The backtrace may be buffered in sys.stderr file object. */ + { + PyObject *result; + + result = PyObject_CallFunction(nxt_py_stderr_flush, NULL); + if (nxt_slow_path(result == NULL)) { + PyErr_Clear(); + return; + } + + Py_DECREF(result); + } +#endif +} diff --git a/src/python/nxt_python.h b/src/python/nxt_python.h new file mode 100644 index 00000000..417df7fd --- /dev/null +++ b/src/python/nxt_python.h @@ -0,0 +1,31 @@ + +/* + * Copyright (C) NGINX, Inc. + */ + +#ifndef _NXT_PYTHON_H_INCLUDED_ +#define _NXT_PYTHON_H_INCLUDED_ + + +#include +#include + + +extern PyObject *nxt_py_application; + +typedef struct { + nxt_str_t string; + PyObject **object_p; +} nxt_python_string_t; + +nxt_int_t nxt_python_init_strings(nxt_python_string_t *pstr); +void nxt_python_done_strings(nxt_python_string_t *pstr); + +void nxt_python_print_exception(void); + +nxt_int_t nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init); +int nxt_python_wsgi_run(nxt_unit_ctx_t *ctx); +void nxt_python_wsgi_done(void); + + +#endif /* _NXT_PYTHON_H_INCLUDED_ */ diff --git a/src/python/nxt_python_wsgi.c b/src/python/nxt_python_wsgi.c index c4b7702e..3371dae6 100644 --- a/src/python/nxt_python_wsgi.c +++ b/src/python/nxt_python_wsgi.c @@ -8,17 +8,15 @@ #include -#include -#include - #include -#include #include #include #include #include #include +#include + #include NXT_PYTHON_MOUNTS_H /* @@ -68,11 +66,7 @@ typedef struct { PyObject_HEAD } nxt_py_error_t; -static nxt_int_t nxt_python_start(nxt_task_t *task, - nxt_process_data_t *data); -static nxt_int_t nxt_python_init_strings(void); static void nxt_python_request_handler(nxt_unit_request_info_t *req); -static void nxt_python_atexit(void); static PyObject *nxt_python_create_environ(nxt_task_t *task); static PyObject *nxt_python_get_environ(nxt_python_run_ctx_t *ctx); @@ -99,7 +93,6 @@ static PyObject *nxt_py_input_readlines(nxt_py_input_t *self, PyObject *args); static PyObject *nxt_py_input_iter(PyObject *self); static PyObject *nxt_py_input_next(PyObject *self); -static void nxt_python_print_exception(void); static int nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes); struct nxt_python_run_ctx_s { @@ -109,22 +102,6 @@ struct nxt_python_run_ctx_s { nxt_unit_request_info_t *req; }; -static uint32_t compat[] = { - NXT_VERNUM, NXT_DEBUG, -}; - - -NXT_EXPORT nxt_app_module_t nxt_app_module = { - sizeof(compat), - compat, - nxt_string("python"), - PY_VERSION, - nxt_python_mounts, - nxt_nitems(nxt_python_mounts), - NULL, - nxt_python_start, -}; - static PyMethodDef nxt_py_start_resp_method[] = { {"unit_start_response", nxt_py_start_resp, METH_VARARGS, ""} @@ -158,22 +135,13 @@ static PyTypeObject nxt_py_input_type = { }; -static PyObject *nxt_py_stderr_flush; -static PyObject *nxt_py_application; static PyObject *nxt_py_start_resp_obj; static PyObject *nxt_py_write_obj; static PyObject *nxt_py_environ_ptyp; -#if PY_MAJOR_VERSION == 3 -static wchar_t *nxt_py_home; -#else -static char *nxt_py_home; -#endif - static PyThreadState *nxt_python_thread_state; static nxt_python_run_ctx_t *nxt_python_run_ctx; - static PyObject *nxt_py_80_str; static PyObject *nxt_py_close_str; static PyObject *nxt_py_content_length_str; @@ -191,11 +159,6 @@ static PyObject *nxt_py_server_port_str; static PyObject *nxt_py_server_protocol_str; static PyObject *nxt_py_wsgi_uri_scheme_str; -typedef struct { - nxt_str_t string; - PyObject **object_p; -} nxt_python_string_t; - static nxt_python_string_t nxt_python_strings[] = { { nxt_string("80"), &nxt_py_80_str }, { nxt_string("close"), &nxt_py_close_str }, @@ -213,136 +176,22 @@ static nxt_python_string_t nxt_python_strings[] = { { nxt_string("SERVER_PORT"), &nxt_py_server_port_str }, { nxt_string("SERVER_PROTOCOL"), &nxt_py_server_protocol_str }, { nxt_string("wsgi.url_scheme"), &nxt_py_wsgi_uri_scheme_str }, + { nxt_null_string, NULL }, }; -static nxt_int_t -nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) +nxt_int_t +nxt_python_wsgi_init(nxt_task_t *task, nxt_unit_init_t *init) { - int rc; - char *nxt_py_module; - size_t len; - PyObject *obj, *pypath, *module; - nxt_unit_ctx_t *unit_ctx; - nxt_unit_init_t python_init; - nxt_common_app_conf_t *app_conf; - nxt_python_app_conf_t *c; -#if PY_MAJOR_VERSION == 3 - char *path; - size_t size; - nxt_int_t pep405; - - static const char pyvenv[] = "/pyvenv.cfg"; - static const char bin_python[] = "/bin/python"; -#endif - - app_conf = data->app; - c = &app_conf->u.python; - - if (c->home != NULL) { - len = nxt_strlen(c->home); - -#if PY_MAJOR_VERSION == 3 - - path = nxt_malloc(len + sizeof(pyvenv)); - if (nxt_slow_path(path == NULL)) { - nxt_alert(task, "Failed to allocate memory"); - return NXT_ERROR; - } - - nxt_memcpy(path, c->home, len); - nxt_memcpy(path + len, pyvenv, sizeof(pyvenv)); - - pep405 = (access(path, R_OK) == 0); - - nxt_free(path); - - if (pep405) { - size = (len + sizeof(bin_python)) * sizeof(wchar_t); - - } else { - size = (len + 1) * sizeof(wchar_t); - } - - nxt_py_home = nxt_malloc(size); - if (nxt_slow_path(nxt_py_home == NULL)) { - nxt_alert(task, "Failed to allocate memory"); - return NXT_ERROR; - } - - if (pep405) { - mbstowcs(nxt_py_home, c->home, len); - mbstowcs(nxt_py_home + len, bin_python, sizeof(bin_python)); - Py_SetProgramName(nxt_py_home); - - } else { - mbstowcs(nxt_py_home, c->home, len + 1); - Py_SetPythonHome(nxt_py_home); - } + PyObject *obj; -#else - nxt_py_home = nxt_malloc(len + 1); - if (nxt_slow_path(nxt_py_home == NULL)) { - nxt_alert(task, "Failed to allocate memory"); - return NXT_ERROR; - } - - nxt_memcpy(nxt_py_home, c->home, len + 1); - Py_SetPythonHome(nxt_py_home); -#endif - } - - Py_InitializeEx(0); - - module = NULL; obj = NULL; - if (nxt_slow_path(nxt_python_init_strings() != NXT_OK)) { + if (nxt_slow_path(nxt_python_init_strings(nxt_python_strings) != NXT_OK)) { nxt_alert(task, "Python failed to init string objects"); goto fail; } - obj = PySys_GetObject((char *) "stderr"); - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to get \"sys.stderr\" object"); - goto fail; - } - - nxt_py_stderr_flush = PyObject_GetAttrString(obj, "flush"); - if (nxt_slow_path(nxt_py_stderr_flush == NULL)) { - nxt_alert(task, "Python failed to get \"flush\" attribute of " - "\"sys.stderr\" object"); - goto fail; - } - - Py_DECREF(obj); - - if (c->path.length > 0) { - obj = PyString_FromStringAndSize((char *) c->path.start, - c->path.length); - - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to create string object \"%V\"", - &c->path); - goto fail; - } - - pypath = PySys_GetObject((char *) "path"); - - if (nxt_slow_path(pypath == NULL)) { - nxt_alert(task, "Python failed to get \"sys.path\" list"); - goto fail; - } - - if (nxt_slow_path(PyList_Insert(pypath, 0, obj) != 0)) { - nxt_alert(task, "Python failed to insert \"%V\" into \"sys.path\"", - &c->path); - goto fail; - } - - Py_DECREF(obj); - } - obj = PyCFunction_New(nxt_py_start_resp_method, NULL); if (nxt_slow_path(obj == NULL)) { nxt_alert(task, @@ -366,107 +215,43 @@ nxt_python_start(nxt_task_t *task, nxt_process_data_t *data) } nxt_py_environ_ptyp = obj; - - obj = Py_BuildValue("[s]", "unit"); - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to create the \"sys.argv\" list"); - goto fail; - } - - if (nxt_slow_path(PySys_SetObject((char *) "argv", obj) != 0)) { - nxt_alert(task, "Python failed to set the \"sys.argv\" list"); - goto fail; - } - - Py_CLEAR(obj); - - nxt_py_module = nxt_alloca(c->module.length + 1); - nxt_memcpy(nxt_py_module, c->module.start, c->module.length); - nxt_py_module[c->module.length] = '\0'; - - module = PyImport_ImportModule(nxt_py_module); - if (nxt_slow_path(module == NULL)) { - nxt_alert(task, "Python failed to import module \"%s\"", nxt_py_module); - nxt_python_print_exception(); - goto fail; - } - - obj = PyDict_GetItemString(PyModule_GetDict(module), "application"); - if (nxt_slow_path(obj == NULL)) { - nxt_alert(task, "Python failed to get \"application\" " - "from module \"%s\"", nxt_py_module); - goto fail; - } - - if (nxt_slow_path(PyCallable_Check(obj) == 0)) { - nxt_alert(task, "\"application\" in module \"%s\" " - "is not a callable object", nxt_py_module); - goto fail; - } - - Py_INCREF(obj); - Py_CLEAR(module); - - nxt_py_application = obj; obj = NULL; - nxt_unit_default_init(task, &python_init); - - python_init.callbacks.request_handler = nxt_python_request_handler; - python_init.shm_limit = data->app->shm_limit; - - unit_ctx = nxt_unit_init(&python_init); - if (nxt_slow_path(unit_ctx == NULL)) { - goto fail; - } - - nxt_python_thread_state = PyEval_SaveThread(); - - rc = nxt_unit_run(unit_ctx); - - nxt_unit_done(unit_ctx); - - PyEval_RestoreThread(nxt_python_thread_state); - - nxt_python_atexit(); - - exit(rc); + init->callbacks.request_handler = nxt_python_request_handler; return NXT_OK; fail: Py_XDECREF(obj); - Py_XDECREF(module); - - nxt_python_atexit(); return NXT_ERROR; } -static nxt_int_t -nxt_python_init_strings(void) +int +nxt_python_wsgi_run(nxt_unit_ctx_t *ctx) { - PyObject *obj; - nxt_uint_t i; - nxt_python_string_t *pstr; + int rc; - for (i = 0; i < nxt_nitems(nxt_python_strings); i++) { - pstr = &nxt_python_strings[i]; + nxt_python_thread_state = PyEval_SaveThread(); - obj = PyString_FromStringAndSize((char *) pstr->string.start, - pstr->string.length); - if (nxt_slow_path(obj == NULL)) { - return NXT_ERROR; - } + rc = nxt_unit_run(ctx); - PyUnicode_InternInPlace(&obj); + PyEval_RestoreThread(nxt_python_thread_state); - *pstr->object_p = obj; - } + return rc; +} - return NXT_OK; + +void +nxt_python_wsgi_done(void) +{ + nxt_python_done_strings(nxt_python_strings); + + Py_XDECREF(nxt_py_start_resp_obj); + Py_XDECREF(nxt_py_write_obj); + Py_XDECREF(nxt_py_environ_ptyp); } @@ -597,29 +382,6 @@ done: } -static void -nxt_python_atexit(void) -{ - nxt_uint_t i; - - for (i = 0; i < nxt_nitems(nxt_python_strings); i++) { - Py_XDECREF(*nxt_python_strings[i].object_p); - } - - Py_XDECREF(nxt_py_stderr_flush); - Py_XDECREF(nxt_py_application); - Py_XDECREF(nxt_py_start_resp_obj); - Py_XDECREF(nxt_py_write_obj); - Py_XDECREF(nxt_py_environ_ptyp); - - Py_Finalize(); - - if (nxt_py_home != NULL) { - nxt_free(nxt_py_home); - } -} - - static PyObject * nxt_python_create_environ(nxt_task_t *task) { @@ -1386,28 +1148,6 @@ nxt_py_input_next(PyObject *self) } -static void -nxt_python_print_exception(void) -{ - PyErr_Print(); - -#if PY_MAJOR_VERSION == 3 - /* The backtrace may be buffered in sys.stderr file object. */ - { - PyObject *result; - - result = PyObject_CallFunction(nxt_py_stderr_flush, NULL); - if (nxt_slow_path(result == NULL)) { - PyErr_Clear(); - return; - } - - Py_DECREF(result); - } -#endif -} - - static int nxt_python_write(nxt_python_run_ctx_t *ctx, PyObject *bytes) { -- cgit