Added SSL/TLS support on connection level.

1 files changed, 372 insertions, 231 deletions

diff --git a/src/nxt_openssl.c b/src/nxt_openssl.c
index d8c892dd..ca0e698c 100644
--- a/src/nxt_openssl.c
+++ b/src/nxt_openssl.c
@@ -12,57 +12,68 @@
 
 typedef struct {
     SSL            *session;
+    nxt_conn_t     *conn;
 
     int            ssl_error;
     uint8_t        times;      /* 2 bits */
+    uint8_t        handshake;  /* 1 bit  */
 
     nxt_buf_mem_t  buffer;
 } nxt_openssl_conn_t;
 
 
-static nxt_int_t nxt_openssl_server_init(nxt_ssltls_conf_t *conf);
+typedef enum {
+    NXT_OPENSSL_HANDSHAKE = 0,
+    NXT_OPENSSL_READ,
+    NXT_OPENSSL_WRITE,
+    NXT_OPENSSL_SHUTDOWN,
+} nxt_openssl_io_t;
 
-static void nxt_openssl_conn_init(nxt_task_t *task, nxt_ssltls_conf_t *conf,
+
+static nxt_int_t nxt_openssl_library_init(nxt_task_t *task);
+static void nxt_openssl_library_free(nxt_task_t *task);
+#if OPENSSL_VERSION_NUMBER < 0x10100004L
+static nxt_int_t nxt_openssl_locks_init(void);
+static void nxt_openssl_lock(int mode, int type, const char *file, int line);
+static unsigned long nxt_openssl_thread_id(void);
+static void nxt_openssl_locks_free(void);
+#endif
+static nxt_int_t nxt_openssl_server_init(nxt_task_t *task,
+    nxt_tls_conf_t *conf);
+static void nxt_openssl_server_free(nxt_task_t *task, nxt_tls_conf_t *conf);
+static void nxt_openssl_conn_init(nxt_task_t *task, nxt_tls_conf_t *conf,
     nxt_conn_t *c);
-static void nxt_openssl_session_cleanup(nxt_task_t *task, void *data);
 static void nxt_openssl_conn_handshake(nxt_task_t *task, void *obj, void *data);
-static void nxt_openssl_conn_io_read(nxt_task_t *task, void *obj, void *data);
+static ssize_t nxt_openssl_conn_io_recvbuf(nxt_conn_t *c, nxt_buf_t *b);
+static ssize_t nxt_openssl_conn_io_sendbuf(nxt_task_t *task, nxt_sendbuf_t *sb);
+static ssize_t nxt_openssl_conn_io_send(nxt_task_t *task, nxt_sendbuf_t *sb,
+    void *buf, size_t size);
 static void nxt_openssl_conn_io_shutdown(nxt_task_t *task, void *obj,
     void *data);
-static ssize_t nxt_openssl_conn_io_write_chunk(nxt_conn_t *c, nxt_buf_t *b,
-    size_t limit);
-static ssize_t nxt_openssl_conn_io_send(nxt_conn_t *c, void *buf, size_t size);
-static nxt_int_t nxt_openssl_conn_test_error(nxt_task_t *task,
-    nxt_conn_t *c, int ret, nxt_err_t sys_err, nxt_work_handler_t handler);
-static void nxt_cdecl nxt_openssl_conn_error(nxt_conn_t *c, nxt_err_t err,
-    const char *fmt, ...);
-static nxt_uint_t nxt_openssl_log_error_level(nxt_conn_t *c, nxt_err_t err);
-static void nxt_cdecl nxt_openssl_log_error(nxt_uint_t level, nxt_log_t *log,
-    const char *fmt, ...);
-static u_char *nxt_openssl_copy_error(u_char *p, u_char *end);
-
-
-const nxt_ssltls_lib_t  nxt_openssl_lib = {
-    nxt_openssl_server_init,
-    NULL,
+static nxt_int_t nxt_openssl_conn_test_error(nxt_task_t *task, nxt_conn_t *c,
+    int ret, nxt_err_t sys_err, nxt_openssl_io_t io);
+static void nxt_cdecl nxt_openssl_conn_error(nxt_task_t *task,
+    nxt_err_t err, const char *fmt, ...);
+static nxt_uint_t nxt_openssl_log_error_level(nxt_err_t err);
+
+
+const nxt_tls_lib_t  nxt_openssl_lib = {
+    .library_init = nxt_openssl_library_init,
+    .library_free = nxt_openssl_library_free,
+
+    .server_init = nxt_openssl_server_init,
+    .server_free = nxt_openssl_server_free,
 };
 
 
 static nxt_conn_io_t  nxt_openssl_conn_io = {
-    NULL,
-    NULL,
-
-    nxt_openssl_conn_io_read,
-    NULL,
-    NULL,
+    .read = nxt_conn_io_read,
+    .recvbuf = nxt_openssl_conn_io_recvbuf,
 
-    nxt_conn_io_write,
-    nxt_openssl_conn_io_write_chunk,
-    NULL,
-    NULL,
-    nxt_openssl_conn_io_send,
+    .write = nxt_conn_io_write,
+    .sendbuf = nxt_openssl_conn_io_sendbuf,
 
-    nxt_openssl_conn_io_shutdown,
+    .shutdown = nxt_openssl_conn_io_shutdown,
 };
 
 
@@ -71,7 +82,7 @@ static int   nxt_openssl_connection_index;
 
 
 static nxt_int_t
-nxt_openssl_start(nxt_thread_t *thr)
+nxt_openssl_library_init(nxt_task_t *task)
 {
     int  index;
 
@@ -79,22 +90,38 @@ nxt_openssl_start(nxt_thread_t *thr)
         return NXT_OK;
     }
 
-    SSL_load_error_strings();
+#if OPENSSL_VERSION_NUMBER >= 0x10100003L
 
-    OPENSSL_config(NULL);
+    OPENSSL_init_ssl(OPENSSL_INIT_LOAD_CONFIG, NULL);
 
-    /*
-     * SSL_library_init(3):
-     *
-     *   SSL_library_init() always returns "1",
-     *   so it is safe to discard the return value.
-     */
-    (void) SSL_library_init();
+#else
+    {
+        nxt_int_t  ret;
+
+        SSL_load_error_strings();
+
+        OPENSSL_config(NULL);
+
+        /*
+         * SSL_library_init(3):
+         *
+         *   SSL_library_init() always returns "1",
+         *   so it is safe to discard the return value.
+         */
+        (void) SSL_library_init();
+
+        ret = nxt_openssl_locks_init();
+        if (nxt_slow_path(ret != NXT_OK)) {
+            return ret;
+        }
+    }
+
+#endif
 
     nxt_openssl_version = SSLeay();
 
-    nxt_log_error(NXT_LOG_INFO, thr->log, "%s, %xl",
-                  SSLeay_version(SSLEAY_VERSION), nxt_openssl_version);
+    nxt_log(task, NXT_LOG_INFO, "%s, %xl",
+            SSLeay_version(SSLEAY_VERSION), nxt_openssl_version);
 
 #ifndef SSL_OP_NO_COMPRESSION
     {
@@ -116,7 +143,7 @@ nxt_openssl_start(nxt_thread_t *thr)
     index = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL);
 
     if (index == -1) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, thr->log,
+        nxt_openssl_log_error(task, NXT_LOG_ALERT,
                               "SSL_get_ex_new_index() failed");
         return NXT_ERROR;
     }
@@ -127,29 +154,115 @@ nxt_openssl_start(nxt_thread_t *thr)
 }
 
 
+#if OPENSSL_VERSION_NUMBER >= 0x10100003L
+
+static void
+nxt_openssl_library_free(nxt_task_t *task)
+{
+}
+
+#else
+
+static nxt_thread_mutex_t  *nxt_openssl_locks;
+
 static nxt_int_t
-nxt_openssl_server_init(nxt_ssltls_conf_t *conf)
+nxt_openssl_locks_init(void)
 {
-    SSL_CTX              *ctx;
-    const char           *certificate, *key, *ciphers, *ca_certificate;
-    nxt_thread_t         *thr;
-    STACK_OF(X509_NAME)  *list;
+    int        i, n;
+    nxt_int_t  ret;
 
-    thr = nxt_thread();
+    n = CRYPTO_num_locks();
 
-    if (nxt_openssl_start(thr) != NXT_OK) {
+    nxt_openssl_locks = OPENSSL_malloc(n * sizeof(nxt_thread_mutex_t));
+    if (nxt_slow_path(nxt_openssl_locks == NULL)) {
         return NXT_ERROR;
     }
 
+    for (i = 0; i < n; i++) {
+        ret = nxt_thread_mutex_create(&nxt_openssl_locks[i]);
+        if (nxt_slow_path(ret != NXT_OK)) {
+            return ret;
+        }
+    }
+
+    CRYPTO_set_locking_callback(nxt_openssl_lock);
+
+    CRYPTO_set_id_callback(nxt_openssl_thread_id);
+
+    return NXT_OK;
+}
+
+
+static void
+nxt_openssl_lock(int mode, int type, const char *file, int line)
+{
+    nxt_thread_mutex_t  *lock;
+
+    lock = &nxt_openssl_locks[type];
+
+    if ((mode & CRYPTO_LOCK) != 0) {
+        (void) nxt_thread_mutex_lock(lock);
+
+    } else {
+        (void) nxt_thread_mutex_unlock(lock);
+    }
+}
+
+
+static u_long
+nxt_openssl_thread_id(void)
+{
+    return (u_long) nxt_thread_handle();
+}
+
+
+static void
+nxt_openssl_library_free(nxt_task_t *task)
+{
+    nxt_openssl_locks_free();
+}
+
+
+static void
+nxt_openssl_locks_free(void)
+{
+    int  i, n;
+
+    n = CRYPTO_num_locks();
+
+    CRYPTO_set_locking_callback(NULL);
+
+    for (i = 0; i < n; i++) {
+        nxt_thread_mutex_destroy(&nxt_openssl_locks[i]);
+    }
+
+    OPENSSL_free(nxt_openssl_locks);
+}
+
+#endif
+
+
+static nxt_int_t
+nxt_openssl_server_init(nxt_task_t *task, nxt_tls_conf_t *conf)
+{
+    SSL_CTX              *ctx;
+    const char           *certificate, *key, *ciphers, *ca_certificate;
+    STACK_OF(X509_NAME)  *list;
+
     ctx = SSL_CTX_new(SSLv23_server_method());
     if (ctx == NULL) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, thr->log, "SSL_CTX_new() failed");
+        nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_CTX_new() failed");
         return NXT_ERROR;
     }
 
     conf->ctx = ctx;
     conf->conn_init = nxt_openssl_conn_init;
 
+#ifdef SSL_OP_NO_RENEGOTIATION
+    /* Renegration is not currently supported. */
+    SSL_CTX_set_options(ctx, SSL_OP_NO_RENEGOTIATION);
+#endif
+
 #ifdef SSL_OP_NO_COMPRESSION
     /*
      * Disable gzip compression in OpenSSL 1.0.0,
@@ -174,7 +287,7 @@ nxt_openssl_server_init(nxt_ssltls_conf_t *conf)
     certificate = conf->certificate;
 
     if (SSL_CTX_use_certificate_chain_file(ctx, certificate) == 0) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, thr->log,
+        nxt_openssl_log_error(task, NXT_LOG_ALERT,
                               "SSL_CTX_use_certificate_file(\"%s\") failed",
                               certificate);
         goto fail;
@@ -183,7 +296,7 @@ nxt_openssl_server_init(nxt_ssltls_conf_t *conf)
     key = conf->certificate_key;
 
     if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) == 0) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, thr->log,
+        nxt_openssl_log_error(task, NXT_LOG_ALERT,
                               "SSL_CTX_use_PrivateKey_file(\"%s\") failed",
                               key);
         goto fail;
@@ -192,7 +305,7 @@ nxt_openssl_server_init(nxt_ssltls_conf_t *conf)
     ciphers = (conf->ciphers != NULL) ? conf->ciphers : "HIGH:!aNULL:!MD5";
 
     if (SSL_CTX_set_cipher_list(ctx, ciphers) == 0) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, thr->log,
+        nxt_openssl_log_error(task, NXT_LOG_ALERT,
                               "SSL_CTX_set_cipher_list(\"%s\") failed",
                               ciphers);
         goto fail;
@@ -211,7 +324,7 @@ nxt_openssl_server_init(nxt_ssltls_conf_t *conf)
         ca_certificate = conf->ca_certificate;
 
         if (SSL_CTX_load_verify_locations(ctx, ca_certificate, NULL) == 0) {
-            nxt_openssl_log_error(NXT_LOG_ALERT, thr->log,
+            nxt_openssl_log_error(task, NXT_LOG_ALERT,
                               "SSL_CTX_load_verify_locations(\"%s\") failed",
                               ca_certificate);
             goto fail;
@@ -220,7 +333,7 @@ nxt_openssl_server_init(nxt_ssltls_conf_t *conf)
         list = SSL_load_client_CA_file(ca_certificate);
 
         if (list == NULL) {
-            nxt_openssl_log_error(NXT_LOG_ALERT, thr->log,
+            nxt_openssl_log_error(task, NXT_LOG_ALERT,
                               "SSL_load_client_CA_file(\"%s\") failed",
                               ca_certificate);
             goto fail;
@@ -246,55 +359,53 @@ fail:
 
 
 static void
-nxt_openssl_conn_init(nxt_task_t *task, nxt_ssltls_conf_t *conf, nxt_conn_t *c)
+nxt_openssl_server_free(nxt_task_t *task, nxt_tls_conf_t *conf)
 {
-    int                     ret;
-    SSL                     *s;
-    SSL_CTX                 *ctx;
-    nxt_openssl_conn_t      *ssltls;
-    nxt_mem_pool_cleanup_t  *mpcl;
+    SSL_CTX_free(conf->ctx);
+}
 
-    nxt_log_debug(c->socket.log, "openssl conn init");
 
-    ssltls = nxt_mp_zget(c->mem_pool, sizeof(nxt_openssl_conn_t));
-    if (ssltls == NULL) {
-        goto fail;
-    }
+static void
+nxt_openssl_conn_init(nxt_task_t *task, nxt_tls_conf_t *conf, nxt_conn_t *c)
+{
+    int                 ret;
+    SSL                 *s;
+    SSL_CTX             *ctx;
+    nxt_openssl_conn_t  *tls;
 
-    c->u.ssltls = ssltls;
-    nxt_buf_mem_set_size(&ssltls->buffer, conf->buffer_size);
+    nxt_log_debug(c->socket.log, "openssl conn init");
 
-    mpcl = nxt_mem_pool_cleanup(c->mem_pool, 0);
-    if (mpcl == NULL) {
+    tls = nxt_mp_zget(c->mem_pool, sizeof(nxt_openssl_conn_t));
+    if (tls == NULL) {
         goto fail;
     }
 
+    c->u.tls = tls;
+    nxt_buf_mem_set_size(&tls->buffer, conf->buffer_size);
+
     ctx = conf->ctx;
 
     s = SSL_new(ctx);
     if (s == NULL) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, c->socket.log,
-                              "SSL_new() failed");
+        nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_new() failed");
         goto fail;
     }
 
-    ssltls->session = s;
-    mpcl->handler = nxt_openssl_session_cleanup;
-    mpcl->data = ssltls;
+    tls->session = s;
+    tls->conn = c;
 
     ret = SSL_set_fd(s, c->socket.fd);
 
     if (ret == 0) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, c->socket.log,
-                              "SSL_set_fd(%d) failed", c->socket.fd);
+        nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_set_fd(%d) failed",
+                              c->socket.fd);
         goto fail;
     }
 
     SSL_set_accept_state(s);
 
     if (SSL_set_ex_data(s, nxt_openssl_connection_index, c) == 0) {
-        nxt_openssl_log_error(NXT_LOG_ALERT, c->socket.log,
-                              "SSL_set_ex_data() failed");
+        nxt_openssl_log_error(task, NXT_LOG_ALERT, "SSL_set_ex_data() failed");
         goto fail;
     }
 
@@ -311,38 +422,37 @@ fail:
 }
 
 
-static void
-nxt_openssl_session_cleanup(nxt_task_t *task, void *data)
+nxt_inline void
+nxt_openssl_conn_free(nxt_task_t *task, nxt_openssl_conn_t *tls)
 {
-    nxt_openssl_conn_t  *ssltls;
-
-    ssltls = data;
-
-    nxt_debug(task, "openssl session cleanup");
+    nxt_debug(task, "openssl conn free");
 
-    nxt_free(ssltls->buffer.start);
+    nxt_free(tls->buffer.start);
 
-    SSL_free(ssltls->session);
+    SSL_free(tls->session);
 }
 
 
 static void
 nxt_openssl_conn_handshake(nxt_task_t *task, void *obj, void *data)
 {
-    int                 ret;
-    nxt_int_t           n;
-    nxt_err_t           err;
-    nxt_conn_t          *c;
-    nxt_openssl_conn_t  *ssltls;
+    int                     ret;
+    nxt_int_t               n;
+    nxt_err_t               err;
+    nxt_conn_t              *c;
+    nxt_work_queue_t        *wq;
+    nxt_work_handler_t      handler;
+    nxt_openssl_conn_t      *tls;
+    const nxt_conn_state_t  *state;
 
     c = obj;
-    ssltls = c->u.ssltls;
+    tls = c->u.tls;
 
-    nxt_debug(task, "openssl conn handshake: %d", ssltls->times);
+    nxt_debug(task, "openssl conn handshake: %d", tls->times);
 
-    /* "ssltls->times == 1" is suitable to run SSL_do_handshake() in job. */
+    /* "tls->times == 1" is suitable to run SSL_do_handshake() in job. */
 
-    ret = SSL_do_handshake(ssltls->session);
+    ret = SSL_do_handshake(tls->session);
 
     err = (ret <= 0) ? nxt_socket_errno : 0;
 
@@ -350,130 +460,161 @@ nxt_openssl_conn_handshake(nxt_task_t *task, void *obj, void *data)
 
     nxt_debug(task, "SSL_do_handshake(%d): %d err:%d", c->socket.fd, ret, err);
 
+    state = (c->read_state != NULL) ? c->read_state : c->write_state;
+
     if (ret > 0) {
         /* ret == 1, the handshake was successfully completed. */
-        nxt_openssl_conn_io_read(task, c, data);
-        return;
-    }
+        tls->handshake = 1;
 
-    n = nxt_openssl_conn_test_error(task, c, ret, err,
-                                    nxt_openssl_conn_handshake);
+        if (c->read_state != NULL) {
+            if (state->io_read_handler != NULL || c->read != NULL) {
+                nxt_conn_read(task->thread->engine, c);
+                return;
+            }
 
-    if (n == NXT_ERROR) {
-        nxt_openssl_conn_error(c, err, "SSL_do_handshake(%d) failed",
-                               c->socket.fd);
+        } else {
+            if (c->write != NULL) {
+                nxt_conn_write(task->thread->engine, c);
+                return;
+            }
+        }
 
-        nxt_work_queue_add(c->read_work_queue, c->read_state->error_handler,
-                           task, c, data);
+        handler = state->ready_handler;
 
-    } else if (ssltls->ssl_error == SSL_ERROR_WANT_READ && ssltls->times < 2) {
-        ssltls->times++;
-    }
-}
+    } else {
+        c->socket.read_handler = nxt_openssl_conn_handshake;
+        c->socket.write_handler = nxt_openssl_conn_handshake;
 
+        n = nxt_openssl_conn_test_error(task, c, ret, err,
+                                        NXT_OPENSSL_HANDSHAKE);
+        switch (n) {
 
-static void
-nxt_openssl_conn_io_read(nxt_task_t *task, void *obj, void *data)
-{
-    int                 ret;
-    nxt_buf_t           *b;
-    nxt_int_t           n;
-    nxt_err_t           err;
-    nxt_conn_t          *c;
-    nxt_work_handler_t  handler;
-    nxt_openssl_conn_t  *ssltls;
+        case NXT_AGAIN:
+            if (tls->ssl_error == SSL_ERROR_WANT_READ && tls->times < 2) {
+                tls->times++;
+            }
 
-    c = obj;
+            return;
 
-    nxt_debug(task, "openssl conn read");
+        case 0:
+            handler = state->close_handler;
+            break;
 
-    handler = c->read_state->ready_handler;
-    b = c->read;
+        default:
+        case NXT_ERROR:
+            c->socket.error = err;
+            nxt_openssl_conn_error(task, err, "SSL_do_handshake(%d) failed",
+                                   c->socket.fd);
 
-    /* b == NULL is used to test descriptor readiness. */
+            handler = state->error_handler;
+            break;
+        }
+    }
 
-    if (b != NULL) {
-        ssltls = c->u.ssltls;
+    wq = (c->read_state != NULL) ? c->read_work_queue : c->write_work_queue;
 
-        ret = SSL_read(ssltls->session, b->mem.free, b->mem.end - b->mem.free);
+    nxt_work_queue_add(wq, handler, task, c, data);
+}
 
-        err = (ret <= 0) ? nxt_socket_errno : 0;
 
-        nxt_debug(task, "SSL_read(%d, %p, %uz): %d err:%d",
-                  c->socket.fd, b->mem.free, b->mem.end - b->mem.free,
-                  ret, err);
+static ssize_t
+nxt_openssl_conn_io_recvbuf(nxt_conn_t *c, nxt_buf_t *b)
+{
+    int                 ret;
+    size_t              size;
+    nxt_int_t           n;
+    nxt_err_t           err;
+    nxt_openssl_conn_t  *tls;
 
-        if (ret > 0) {
-            /* c->socket.read_ready is kept. */
-            b->mem.free += ret;
-            handler = c->read_state->ready_handler;
+    tls = c->u.tls;
+    size = b->mem.end - b->mem.free;
 
-        } else {
-            n = nxt_openssl_conn_test_error(task, c, ret, err,
-                                            nxt_openssl_conn_io_read);
+    ret = SSL_read(tls->session, b->mem.free, size);
 
-            if (nxt_fast_path(n != NXT_ERROR)) {
-                return;
-            }
+    err = (ret <= 0) ? nxt_socket_errno : 0;
 
-            nxt_openssl_conn_error(c, err, "SSL_read(%d, %p, %uz) failed",
-                                   c->socket.fd, b->mem.free,
-                                   b->mem.end - b->mem.free);
+    nxt_debug(c->socket.task, "SSL_read(%d, %p, %uz): %d err:%d",
+              c->socket.fd, b->mem.free, size, ret, err);
 
-            handler = c->read_state->error_handler;
+    if (ret > 0) {
+        if ((size_t) ret < size) {
+            c->socket.read_ready = 0;
         }
+
+        return ret;
     }
 
-    nxt_work_queue_add(c->read_work_queue, handler, task, c, data);
+    n = nxt_openssl_conn_test_error(c->socket.task, c, ret, err,
+                                    NXT_OPENSSL_READ);
+    if (n == NXT_ERROR) {
+        c->socket.error = err;
+        nxt_openssl_conn_error(c->socket.task, err,
+                               "SSL_read(%d, %p, %uz) failed",
+                               c->socket.fd, b->mem.free, size);
+    }
+
+    return n;
 }
 
 
 static ssize_t
-nxt_openssl_conn_io_write_chunk(nxt_conn_t *c, nxt_buf_t *b, size_t limit)
+nxt_openssl_conn_io_sendbuf(nxt_task_t *task, nxt_sendbuf_t *sb)
 {
-    nxt_openssl_conn_t  *ssltls;
+    nxt_uint_t    niov;
+    struct iovec  iov;
 
-    nxt_debug(c->socket.task, "openssl conn write chunk");
+    niov = nxt_sendbuf_mem_coalesce0(task, sb, &iov, 1);
 
-    ssltls = c->u.ssltls;
+    if (niov == 0 && sb->sync) {
+        return 0;
+    }
 
-    return nxt_sendbuf_copy_coalesce(c, &ssltls->buffer, b, limit);
+    return nxt_openssl_conn_io_send(task, sb, iov.iov_base, iov.iov_len);
 }
 
 
 static ssize_t
-nxt_openssl_conn_io_send(nxt_conn_t *c, void *buf, size_t size)
+nxt_openssl_conn_io_send(nxt_task_t *task, nxt_sendbuf_t *sb, void *buf,
+    size_t size)
 {
     int                 ret;
     nxt_err_t           err;
     nxt_int_t           n;
-    nxt_openssl_conn_t  *ssltls;
+    nxt_conn_t          *c;
+    nxt_openssl_conn_t  *tls;
 
-    ssltls = c->u.ssltls;
+    tls = sb->tls;
 
-    ret = SSL_write(ssltls->session, buf, size);
+    ret = SSL_write(tls->session, buf, size);
 
     if (ret <= 0) {
         err = nxt_socket_errno;
-        c->socket.error = err;
+        sb->error = err;
 
     } else {
         err = 0;
     }
 
-    nxt_log_debug(c->socket.log, "SSL_write(%d, %p, %uz): %d err:%d",
-                  c->socket.fd, buf, size, ret, err);
+    nxt_debug(task, "SSL_write(%d, %p, %uz): %d err:%d",
+              sb->socket, buf, size, ret, err);
 
     if (ret > 0) {
         return ret;
     }
 
-    n = nxt_openssl_conn_test_error(c->socket.task, c, ret, err,
-                                    nxt_conn_io_write);
+    c = tls->conn;
+    c->socket.write_ready = sb->ready;
+    c->socket.error = sb->error;
+
+    n = nxt_openssl_conn_test_error(task, c, ret, err, NXT_OPENSSL_WRITE);
+
+    sb->ready = c->socket.write_ready;
+    sb->error = c->socket.error;
 
     if (n == NXT_ERROR) {
-        nxt_openssl_conn_error(c, err, "SSL_write(%d, %p, %uz) failed",
-                               c->socket.fd, buf, size);
+        sb->error = err;
+        nxt_openssl_conn_error(task, err, "SSL_write(%d, %p, %uz) failed",
+                               sb->socket, buf, size);
     }
 
     return n;
@@ -489,18 +630,19 @@ nxt_openssl_conn_io_shutdown(nxt_task_t *task, void *obj, void *data)
     nxt_int_t           n;
     nxt_bool_t          quiet, once;
     nxt_conn_t          *c;
+    nxt_openssl_conn_t  *tls;
     nxt_work_handler_t  handler;
-    nxt_openssl_conn_t  *ssltls;
 
     c = obj;
 
     nxt_debug(task, "openssl conn shutdown");
 
-    ssltls = c->u.ssltls;
-    s = ssltls->session;
+    c->read_state = NULL;
+    tls = c->u.tls;
+    s = tls->session;
 
-    if (s == NULL) {
-        handler = c->write_state->close_handler;
+    if (s == NULL || !tls->handshake) {
+        handler = c->write_state->ready_handler;
         goto done;
     }
 
@@ -532,22 +674,22 @@ nxt_openssl_conn_io_shutdown(nxt_task_t *task, void *obj, void *data)
 
         if (ret > 0) {
             /* ret == 1, the shutdown was successfully completed. */
-            handler = c->write_state->close_handler;
+            handler = c->write_state->ready_handler;
             goto done;
         }
 
         if (ret == 0) {
             /*
              * If SSL_shutdown() returns 0 then it should be called
-             * again.  The second SSL_shutdown() call should returns
+             * again.  The second SSL_shutdown() call should return
              * -1/SSL_ERROR_WANT_READ or -1/SSL_ERROR_WANT_WRITE.
              * OpenSSL prior to 0.9.8m version however never returns
-             * -1 at all.  Fortunately, OpenSSL internals preserve
+             * -1 at all.  Fortunately, OpenSSL preserves internally
              * correct status available via SSL_get_error(-1).
              */
             if (once) {
-                mode = SSL_get_shutdown(s);
                 once = 0;
+                mode = SSL_get_shutdown(s);
                 continue;
             }
 
@@ -559,71 +701,82 @@ nxt_openssl_conn_io_shutdown(nxt_task_t *task, void *obj, void *data)
         break;
     }
 
-    n = nxt_openssl_conn_test_error(task, c, ret, err,
-                                    nxt_openssl_conn_io_shutdown);
+    c->socket.read_handler = nxt_openssl_conn_io_shutdown;
+    c->socket.write_handler = nxt_openssl_conn_io_shutdown;
+    c->socket.error_handler = c->write_state->error_handler;
 
-    if (nxt_fast_path(n == 0)) {
-        return;
-    }
+    n = nxt_openssl_conn_test_error(task, c, ret, err, NXT_OPENSSL_SHUTDOWN);
+
+    switch (n) {
+
+    case 0:
+        handler = c->write_state->close_handler;
+        break;
 
-    if (n != NXT_ERROR) {  /* n == NXT_AGAIN */
-        c->socket.error_handler = c->read_state->error_handler;
+    case NXT_AGAIN:
         nxt_timer_add(task->thread->engine, &c->read_timer, 5000);
         return;
-    }
-
-    nxt_openssl_conn_error(c, err, "SSL_shutdown(%d) failed", c->socket.fd);
 
-    handler = c->write_state->error_handler;
+    default:
+    case NXT_ERROR:
+        c->socket.error = err;
+        nxt_openssl_conn_error(task, err, "SSL_shutdown(%d) failed",
+                               c->socket.fd);
+        handler = c->write_state->error_handler;
+    }
 
 done:
 
+    nxt_openssl_conn_free(task, tls);
+
     nxt_work_queue_add(c->write_work_queue, handler, task, c, data);
 }
 
 
 static nxt_int_t
 nxt_openssl_conn_test_error(nxt_task_t *task, nxt_conn_t *c, int ret,
-    nxt_err_t sys_err, nxt_work_handler_t handler)
+    nxt_err_t sys_err, nxt_openssl_io_t io)
 {
     u_long              lib_err;
-    nxt_work_queue_t    *wq;
-    nxt_openssl_conn_t  *ssltls;
+    nxt_openssl_conn_t  *tls;
 
-    ssltls = c->u.ssltls;
+    tls = c->u.tls;
 
-    ssltls->ssl_error = SSL_get_error(ssltls->session, ret);
+    tls->ssl_error = SSL_get_error(tls->session, ret);
 
-    nxt_log_debug(c->socket.log, "SSL_get_error(): %d", ssltls->ssl_error);
+    nxt_debug(task, "SSL_get_error(): %d", tls->ssl_error);
 
-    switch (ssltls->ssl_error) {
+    switch (tls->ssl_error) {
 
     case SSL_ERROR_WANT_READ:
-        nxt_fd_event_block_write(task->thread->engine, &c->socket);
 
-        c->socket.read_ready = 0;
-        c->socket.read_handler = handler;
+        if (io != NXT_OPENSSL_READ) {
+            nxt_fd_event_block_write(task->thread->engine, &c->socket);
+
+            c->socket.read_ready = 0;
 
-        if (nxt_fd_event_is_disabled(c->socket.read)) {
-            nxt_fd_event_enable_read(task->thread->engine, &c->socket);
+            if (nxt_fd_event_is_disabled(c->socket.read)) {
+                nxt_fd_event_enable_read(task->thread->engine, &c->socket);
+            }
         }
 
         return NXT_AGAIN;
 
     case SSL_ERROR_WANT_WRITE:
-        nxt_fd_event_block_read(task->thread->engine, &c->socket);
 
-        c->socket.write_ready = 0;
-        c->socket.write_handler = handler;
+        if (io != NXT_OPENSSL_WRITE) {
+            nxt_fd_event_block_read(task->thread->engine, &c->socket);
 
-        if (nxt_fd_event_is_disabled(c->socket.write)) {
-            nxt_fd_event_enable_write(task->thread->engine, &c->socket);
+            c->socket.write_ready = 0;
+
+            if (nxt_fd_event_is_disabled(c->socket.write)) {
+                nxt_fd_event_enable_write(task->thread->engine, &c->socket);
+            }
         }
 
         return NXT_AGAIN;
 
     case SSL_ERROR_SYSCALL:
-
         lib_err = ERR_peek_error();
 
         nxt_debug(task, "ERR_peek_error(): %l", lib_err);
@@ -634,23 +787,10 @@ nxt_openssl_conn_test_error(nxt_task_t *task, nxt_conn_t *c, int ret,
 
         /* A connection was just closed. */
         c->socket.closed = 1;
-
-        /* Fall through. */
+        return 0;
 
     case SSL_ERROR_ZERO_RETURN:
         /* A "close notify" alert. */
-
-        if (c->read_state != NULL) {
-            wq = c->read_work_queue;
-            handler = c->read_state->close_handler;
-
-        } else {
-            wq = c->write_work_queue;
-            handler = c->write_state->close_handler;
-        }
-
-        nxt_work_queue_add(wq, handler, task, c, c->socket.data);
-
         return 0;
 
     default: /* SSL_ERROR_SSL, etc. */
@@ -661,17 +801,16 @@ nxt_openssl_conn_test_error(nxt_task_t *task, nxt_conn_t *c, int ret,
 
 
 static void nxt_cdecl
-nxt_openssl_conn_error(nxt_conn_t *c, nxt_err_t err, const char *fmt, ...)
+nxt_openssl_conn_error(nxt_task_t *task, nxt_err_t err, const char *fmt, ...)
 {
     u_char      *p, *end;
     va_list     args;
     nxt_uint_t  level;
     u_char      msg[NXT_MAX_ERROR_STR];
 
-    c->socket.error = err;
-    level = nxt_openssl_log_error_level(c, err);
+    level = nxt_openssl_log_error_level(err);
 
-    if (nxt_log_level_enough(c->socket.log, level)) {
+    if (nxt_log_level_enough(task->log, level)) {
 
         end = msg + sizeof(msg);
 
@@ -685,7 +824,7 @@ nxt_openssl_conn_error(nxt_conn_t *c, nxt_err_t err, const char *fmt, ...)
 
         p = nxt_openssl_copy_error(p, end);
 
-        nxt_log_error(level, c->socket.log, "%*s", p - msg, msg);
+        nxt_log(task, level, "%*s", p - msg, msg);
 
     } else {
         ERR_clear_error();
@@ -694,7 +833,7 @@ nxt_openssl_conn_error(nxt_conn_t *c, nxt_err_t err, const char *fmt, ...)
 
 
 static nxt_uint_t
-nxt_openssl_log_error_level(nxt_conn_t *c, nxt_err_t err)
+nxt_openssl_log_error_level(nxt_err_t err)
 {
     switch (ERR_GET_REASON(ERR_peek_error())) {
 
@@ -707,7 +846,9 @@ nxt_openssl_log_error_level(nxt_conn_t *c, nxt_err_t err)
     case SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST:             /*  151 */
     case SSL_R_EXCESSIVE_MESSAGE_SIZE:                    /*  152 */
     case SSL_R_LENGTH_MISMATCH:                           /*  159 */
+#ifdef SSL_R_NO_CIPHERS_PASSED
     case SSL_R_NO_CIPHERS_PASSED:                         /*  182 */
+#endif
     case SSL_R_NO_CIPHERS_SPECIFIED:                      /*  183 */
     case SSL_R_NO_COMPRESSION_SPECIFIED:                  /*  187 */
     case SSL_R_NO_SHARED_CIPHER:                          /*  193 */
@@ -768,8 +909,8 @@ nxt_openssl_log_error_level(nxt_conn_t *c, nxt_err_t err)
 }
 
 
-static void nxt_cdecl
-nxt_openssl_log_error(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...)
+void nxt_cdecl
+nxt_openssl_log_error(nxt_task_t *task, nxt_uint_t level, const char *fmt, ...)
 {
     u_char   *p, *end;
     va_list  args;
@@ -783,11 +924,11 @@ nxt_openssl_log_error(nxt_uint_t level, nxt_log_t *log, const char *fmt, ...)
 
     p = nxt_openssl_copy_error(p, end);
 
-    nxt_log_error(level, log, "%*s", p - msg, msg);
+    nxt_log(task, level, "%*s", p - msg, msg);
 }
 
 
-static u_char *
+u_char *
 nxt_openssl_copy_error(u_char *p, u_char *end)
 {
     int         flags;
@@ -806,8 +947,8 @@ nxt_openssl_copy_error(u_char *p, u_char *end)
     p = nxt_sprintf(p, end, " (%d: %s) (OpenSSL: ", ERR_GET_REASON(err), data);
 
     /*
-     * ... followed by all queued cumbersome OpenSSL
-     * error messages and drain the error queue.
+     * ... followed by all queued cumbersome OpenSSL error messages
+     * and drain the error queue.
      */
     delimiter = "";
     clear = 0;