From 4071de5797da0a104b4983fc1ad393d3744b6128 Mon Sep 17 00:00:00 2001 From: Andrew Clayton Date: Tue, 11 Jun 2024 00:24:28 +0100 Subject: [WIP] HTTP Compression Support --- auto/compression | 96 ++++++++ auto/help | 4 + auto/options | 8 + auto/sources | 16 ++ auto/summary | 3 + configure | 7 +- src/nxt_brotli.c | 86 +++++++ src/nxt_conf_validation.c | 106 ++++++++ src/nxt_http.h | 1 + src/nxt_http_compression.c | 586 +++++++++++++++++++++++++++++++++++++++++++++ src/nxt_http_compression.h | 91 +++++++ src/nxt_http_request.c | 14 ++ src/nxt_http_route.c | 2 + src/nxt_http_static.c | 6 + src/nxt_router.c | 10 + src/nxt_zlib.c | 79 ++++++ src/nxt_zstd.c | 73 ++++++ 17 files changed, 1186 insertions(+), 2 deletions(-) create mode 100644 auto/compression create mode 100644 src/nxt_brotli.c create mode 100644 src/nxt_http_compression.c create mode 100644 src/nxt_http_compression.h create mode 100644 src/nxt_zlib.c create mode 100644 src/nxt_zstd.c diff --git a/auto/compression b/auto/compression new file mode 100644 index 00000000..684242f2 --- /dev/null +++ b/auto/compression @@ -0,0 +1,96 @@ + +# Copyright (C) Alejandro Colomar +# Copyright (C) Andrew Clayton +# Copyright (C) NGINX, Inc. + + +NXT_HAVE_ZLIB=no +NXT_ZLIB_CFLAGS= +NXT_ZLIB_LIBS= + +NXT_HAVE_ZSTD=no +NXT_ZSTD_CFLAGS= +NXT_ZSTD_LIBS= + +NXT_HAVE_BROTLI=no +NXT_BROTLI_CFLAGS= +NXT_BROTLI_LIBS= + + +if [ $NXT_ZLIB = YES ]; then + NXT_ZLIB_CFLAGS="$(pkgconf --cflags-only-I zlib 2>/dev/null || echo "")" + NXT_ZLIB_LIBS="$(pkgconf --libs zlib 2>/dev/null || echo "-lz")" + + nxt_feature="zlib" + nxt_feature_name=NXT_HAVE_ZLIB + nxt_feature_run=no + nxt_feature_incs=$NXT_ZLIB_CFLAGS + nxt_feature_libs=$NXT_ZLIB_LIBS + nxt_feature_test="#include + + #include + + int main(void) { + puts(zlibVersion()); + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_ZLIB=YES + echo " + zlib version: $(pkgconf --modversion zlib)" + fi +fi + + +if [ $NXT_ZSTD = YES ]; then + NXT_ZSTD_CFLAGS="$(pkgconf --cflags-only-I libzstd 2>/dev/null || echo "")" + NXT_ZSTD_LIBS="$(pkgconf --libs libzstd 2>/dev/null || echo "-lzstd")" + + nxt_feature="zstd" + nxt_feature_name=NXT_HAVE_ZSTD + nxt_feature_run=no + nxt_feature_incs=$NXT_ZSTD_CFLAGS + nxt_feature_libs=$NXT_ZSTD_LIBS + nxt_feature_test="#include + + #include + + int main(void) { + printf(\"zstd version: %u\n\", ZSTD_versionNumber()); + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_ZSTD=YES + echo " + zstd version: $(pkgconf --modversion libzstd)" + fi +fi + + +if [ $NXT_BROTLI = YES ]; then + NXT_BROTLI_CFLAGS="$(pkgconf --cflags-only-I libbrotlienc 2>/dev/null || echo "")" + NXT_BROTLI_LIBS="$(pkgconf --libs libbrotlienc 2>/dev/null || echo "-lbrotlienc")" + + nxt_feature="brotli" + nxt_feature_name=NXT_HAVE_BROTLI + nxt_feature_run=no + nxt_feature_incs=$NXT_BROTLI_CFLAGS + nxt_feature_libs=$NXT_BROTLI_LIBS + nxt_feature_test="#include + + #include + + int main(void) { + printf(\"brotli version: %d\n\", + BrotliEncoderVersion()); + return 0; + }" + . auto/feature + + if [ $nxt_found = yes ]; then + NXT_HAVE_BROTLI=YES + echo " + brotli version: $(pkgconf --modversion libbrotlienc)" + fi +fi diff --git a/auto/help b/auto/help index 8f7553f3..0c0d0319 100644 --- a/auto/help +++ b/auto/help @@ -50,6 +50,10 @@ cat << END --openssl enable OpenSSL library usage + --zlib enable zlib compression + --zstd enable zstd compression + --brotli enable brotli compression + --njs enable njs library usage --debug enable debug logging diff --git a/auto/options b/auto/options index 0550c699..3becefb8 100644 --- a/auto/options +++ b/auto/options @@ -26,6 +26,10 @@ NXT_GNUTLS=NO NXT_CYASSL=NO NXT_POLARSSL=NO +NXT_ZLIB=NO +NXT_ZSTD=NO +NXT_BROTLI=NO + NXT_NJS=NO NXT_TEST_BUILD_EPOLL=NO @@ -109,6 +113,10 @@ do --cyassl) NXT_CYASSL=YES ;; --polarssl) NXT_POLARSSL=YES ;; + --zlib) NXT_ZLIB=YES ;; + --zstd) NXT_ZSTD=YES ;; + --brotli) NXT_BROTLI=YES ;; + --njs) NXT_NJS=YES ;; --test-build-epoll) NXT_TEST_BUILD_EPOLL=YES ;; diff --git a/auto/sources b/auto/sources index f34d7fd7..3feafa2a 100644 --- a/auto/sources +++ b/auto/sources @@ -107,6 +107,7 @@ NXT_LIB_SRCS=" \ src/nxt_http_websocket.c \ src/nxt_h1proto_websocket.c \ src/nxt_fs.c \ + src/nxt_http_compression.c \ " @@ -211,6 +212,21 @@ if [ $NXT_POLARSSL = YES ]; then fi +if [ "$NXT_HAVE_ZLIB" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zlib.c" +fi + + +if [ "$NXT_HAVE_ZSTD" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_zstd.c" +fi + + +if [ "$NXT_HAVE_BROTLI" = "YES" ]; then + NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_brotli.c" +fi + + if [ "$NXT_REGEX" = "YES" ]; then if [ "$NXT_HAVE_PCRE2" = "YES" ]; then NXT_LIB_SRCS="$NXT_LIB_SRCS $NXT_LIB_PCRE2_SRCS" diff --git a/auto/summary b/auto/summary index dd7a60a0..d7238152 100644 --- a/auto/summary +++ b/auto/summary @@ -28,6 +28,9 @@ Unit configuration summary: IPv6 support: .............. $NXT_INET6 Unix domain sockets support: $NXT_UNIX_DOMAIN TLS support: ............... $NXT_OPENSSL + zlib support: .............. $NXT_ZLIB + zstd support: .............. $NXT_ZSTD + brotli support: ............ $NXT_BROTLI Regex support: ............. $NXT_REGEX njs support: ............... $NXT_NJS diff --git a/configure b/configure index 2cb4d457..ed256f84 100755 --- a/configure +++ b/configure @@ -127,6 +127,7 @@ NXT_LIBRT= . auto/unix . auto/os/conf . auto/ssltls +. auto/compression if [ $NXT_REGEX = YES ]; then . auto/pcre @@ -168,11 +169,13 @@ END NXT_LIB_AUX_CFLAGS="$NXT_OPENSSL_CFLAGS $NXT_GNUTLS_CFLAGS \\ $NXT_CYASSL_CFLAGS $NXT_POLARSSL_CFLAGS \\ - $NXT_PCRE_CFLAGS" + $NXT_PCRE_CFLAGS $NXT_ZLIB_CFLAGS $NXT_ZSTD_CFLAGS \\ + $NXT_BROTLI_CFLAGS" NXT_LIB_AUX_LIBS="$NXT_OPENSSL_LIBS $NXT_GNUTLS_LIBS \\ $NXT_CYASSL_LIBS $NXT_POLARSSL_LIBS \\ - $NXT_PCRE_LIB" + $NXT_PCRE_LIB $NXT_ZLIB_LIBS $NXT_ZSTD_LIBS \\ + $NXT_BROTLI_LIBS" if [ $NXT_NJS != NO ]; then . auto/njs diff --git a/src/nxt_brotli.c b/src/nxt_brotli.c new file mode 100644 index 00000000..0c9b3658 --- /dev/null +++ b/src/nxt_brotli.c @@ -0,0 +1,86 @@ +/* + * + */ + +/* XXX Remove */ +#define _GNU_SOURCE +#include + + +#include +#include +#include + +#include + +#include + +static void nxt_brotli_free(const nxt_http_comp_compressor_ctx_t *ctx) +{ + BrotliEncoderState *brotli = ctx->brotli_ctx; + + BrotliEncoderDestroyInstance(brotli); +} + +static void nxt_brotli_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + BrotliEncoderState **brotli = &ctx->brotli_ctx; + + *brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL); + BrotliEncoderSetParameter(*brotli, BROTLI_PARAM_QUALITY, ctx->level); + + printf("%7d %s: brotli compression level [%d]\n", gettid(), __func__, + ctx->level); +} + +static size_t nxt_brotli_compressed_size(const nxt_http_comp_compressor_ctx_t *ctx, + size_t in_len) +{ + return BrotliEncoderMaxCompressedSize(in_len); +} + +static ssize_t nxt_brotli_compress(nxt_http_comp_compressor_ctx_t *ctx, + const uint8_t *in_buf, size_t in_len, + uint8_t *out_buf, size_t out_len, bool last) +{ + bool ok; + size_t out_bytes; + uint8_t *outp; + BrotliEncoderState *brotli = ctx->brotli_ctx; + + printf("%7d %s: last/%s\n", gettid(), __func__, last ? "true" : "false"); + printf("%7d %s: in_len [%lu] out_len [%lu]\n", gettid(), __func__, + in_len, out_len); + + outp = out_buf; + + ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_PROCESS, + &in_len, &in_buf, &out_bytes, &outp, + NULL); + + ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FLUSH, + &in_len, &in_buf, &out_bytes, &outp, + NULL); + + printf("%7d %s: in_len [%lu] out_len [%lu] out_bytes [%lu]\n", gettid(), + __func__, in_len, out_len, out_bytes); + if (last) { + ok = BrotliEncoderCompressStream(brotli, BROTLI_OPERATION_FINISH, + &in_len, &in_buf, &out_bytes, &outp, + NULL); + nxt_brotli_free(ctx); + } + + printf("%7d %s: in_len [%lu] out_len [%lu] out_bytes [%lu]\n", gettid(), + __func__, in_len, out_len, out_bytes); + printf("%7d %s: buf [%p] outp [%p]\n", gettid(), __func__, out_buf, outp); + + return out_len - out_bytes; +} + +const nxt_http_comp_operations_t nxt_comp_brotli_ops = { + .init = nxt_brotli_init, + .compressed_size = nxt_brotli_compressed_size, + .deflate = nxt_brotli_compress, + .free_ctx = nxt_brotli_free, +}; diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 4aaa1b9a..e3a75a29 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -138,6 +139,14 @@ static nxt_int_t nxt_conf_vldt_threads(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_compressors(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_compression(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value); +static nxt_int_t nxt_conf_vldt_compression_encoding(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); +static nxt_int_t nxt_conf_vldt_compression_level(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data); static nxt_int_t nxt_conf_vldt_routes_member(nxt_conf_validation_t *vldt, @@ -242,6 +251,8 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_setting_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_websocket_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_compression_members[]; +static nxt_conf_vldt_object_t nxt_conf_vldt_compressor_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_forwarded_members[]; static nxt_conf_vldt_object_t nxt_conf_vldt_client_ip_members[]; #if (NXT_TLS) @@ -368,6 +379,11 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("server_version"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("compression"), + .type = NXT_CONF_VLDT_OBJECT, + .validator = nxt_conf_vldt_object, + .u.members = nxt_conf_vldt_compression_members, }, NXT_CONF_VLDT_END @@ -402,6 +418,40 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_static_members[] = { }; +static nxt_conf_vldt_object_t nxt_conf_vldt_compression_members[] = { + { + .name = nxt_string("types"), + .type = NXT_CONF_VLDT_STRING | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_match_patterns, + }, { + .name = nxt_string("compressors"), + .type = NXT_CONF_VLDT_OBJECT | NXT_CONF_VLDT_ARRAY, + .validator = nxt_conf_vldt_compressors, + }, + + NXT_CONF_VLDT_END +}; + + +static nxt_conf_vldt_object_t nxt_conf_vldt_compressor_members[] = { + { + .name = nxt_string("encoding"), + .type = NXT_CONF_VLDT_STRING, + .flags = NXT_CONF_VLDT_REQUIRED, + .validator = nxt_conf_vldt_compression_encoding, + }, { + .name = nxt_string("level"), + .type = NXT_CONF_VLDT_INTEGER, + .validator = nxt_conf_vldt_compression_level, + }, { + .name = nxt_string("min_length"), + .type = NXT_CONF_VLDT_INTEGER, + }, + + NXT_CONF_VLDT_END +}; + + static nxt_conf_vldt_object_t nxt_conf_vldt_listener_members[] = { { .name = nxt_string("pass"), @@ -2112,6 +2162,62 @@ nxt_conf_vldt_thread_stack_size(nxt_conf_validation_t *vldt, } +static nxt_int_t +nxt_conf_vldt_compressors(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, + void *data) +{ + if (nxt_conf_type(value) == NXT_CONF_ARRAY) { + return nxt_conf_vldt_array_iterator(vldt, value, + &nxt_conf_vldt_compression); + } + + /* NXT_CONF_OBJECT */ + + return nxt_conf_vldt_object_iterator(vldt, value, + &nxt_conf_vldt_compressor_members); +} + + +static nxt_int_t +nxt_conf_vldt_compression(nxt_conf_validation_t *vldt, nxt_conf_value_t *value) +{ + if (nxt_conf_type(value) != NXT_CONF_OBJECT) { + return nxt_conf_vldt_error(vldt, + "The \"compressors\" array must contain " + "only object values."); + } + + return nxt_conf_vldt_object(vldt, value, nxt_conf_vldt_compressor_members); +} + + +static nxt_int_t +nxt_conf_vldt_compression_encoding(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + nxt_str_t token; + + nxt_conf_get_string(value, &token); + + if (nxt_http_comp_compressor_is_valid(&token)) { + return NXT_OK; + } + + return nxt_conf_vldt_error(vldt, "\"%V\" is not a supported compressor.", + &token); +} + + +static nxt_int_t +nxt_conf_vldt_compression_level(nxt_conf_validation_t *vldt, + nxt_conf_value_t *value, void *data) +{ + /* XXX Fill me in */ + + return NXT_OK; +} + + static nxt_int_t nxt_conf_vldt_routes(nxt_conf_validation_t *vldt, nxt_conf_value_t *value, void *data) diff --git a/src/nxt_http.h b/src/nxt_http.h index 5fab5c67..a8ada3f4 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -110,6 +110,7 @@ typedef struct { nxt_http_field_t *content_type; nxt_http_field_t *content_length; nxt_off_t content_length_n; + const nxt_str_t *mime_type; } nxt_http_response_t; diff --git a/src/nxt_http_compression.c b/src/nxt_http_compression.c new file mode 100644 index 00000000..dd91d890 --- /dev/null +++ b/src/nxt_http_compression.c @@ -0,0 +1,586 @@ +/* + * + */ + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#define NXT_COMP_LEVEL_UNSET INT8_MIN + +typedef enum nxt_http_comp_scheme_e nxt_http_comp_scheme_t; +typedef struct nxt_http_comp_type_s nxt_http_comp_type_t; +typedef struct nxt_http_comp_opts_s nxt_http_comp_opts_t; +typedef struct nxt_http_comp_compressor_s nxt_http_comp_compressor_t; +typedef struct nxt_http_comp_ctx_s nxt_http_comp_ctx_t; + +enum nxt_http_comp_scheme_e { + NXT_HTTP_COMP_SCHEME_IDENTITY = 0, +#if NXT_HAVE_ZLIB + NXT_HTTP_COMP_SCHEME_DEFLATE, + NXT_HTTP_COMP_SCHEME_GZIP, +#endif +#if NXT_HAVE_ZSTD + NXT_HTTP_COMP_SCHEME_ZSTD, +#endif +#if NXT_HAVE_BROTLI + NXT_HTTP_COMP_SCHEME_BROTLI, +#endif + + /* keep last */ + NXT_HTTP_COMP_SCHEME_UNKNOWN +}; +#define NXT_NR_COMPRESSORS NXT_HTTP_COMP_SCHEME_UNKNOWN + +struct nxt_http_comp_type_s { + nxt_str_t token; + nxt_http_comp_scheme_t scheme; + int8_t def_compr; + + const nxt_http_comp_operations_t *cops; +}; + +struct nxt_http_comp_opts_s { + int8_t level; + nxt_off_t min_len; +}; + +struct nxt_http_comp_compressor_s { + const nxt_http_comp_type_t *type; + nxt_http_comp_opts_t opts; +}; + +struct nxt_http_comp_ctx_s { + nxt_uint_t idx; + + nxt_off_t resp_clen; + nxt_off_t clen_procd; + + nxt_http_comp_compressor_ctx_t ctx; +}; + +static nxt_tstr_t *accept_encoding_query; +static nxt_http_route_rule_t *mime_types_rule; +static nxt_http_comp_compressor_t *enabled_compressors; +static nxt_uint_t nr_enabled_compressors; + +static nxt_thread_declare_data(nxt_http_comp_ctx_t, compressor_ctx); + +static const nxt_conf_map_t compressors_opts_map[] = { + { + nxt_string("level"), + NXT_CONF_MAP_INT, + offsetof(nxt_http_comp_opts_t, level), + }, { + nxt_string("min_length"), + NXT_CONF_MAP_SIZE, + offsetof(nxt_http_comp_opts_t, min_len), + }, +}; + +static const nxt_http_comp_type_t compressors[] = { + /* Keep this first */ + { + .token = nxt_string("identity"), + .scheme = NXT_HTTP_COMP_SCHEME_IDENTITY, + }, +#if NXT_HAVE_ZLIB + { + .token = nxt_string("deflate"), + .scheme = NXT_HTTP_COMP_SCHEME_DEFLATE, + .def_compr = NXT_HTTP_COMP_ZLIB_DEFAULT_LEVEL, + .cops = &nxt_comp_deflate_ops, + }, { + .token = nxt_string("gzip"), + .scheme = NXT_HTTP_COMP_SCHEME_GZIP, + .def_compr = NXT_HTTP_COMP_ZLIB_DEFAULT_LEVEL, + .cops = &nxt_comp_gzip_ops, + }, +#endif +#if NXT_HAVE_ZSTD + { + .token = nxt_string("zstd"), + .scheme = NXT_HTTP_COMP_SCHEME_ZSTD, + .def_compr = NXT_HTTP_COMP_ZSTD_DEFAULT_LEVEL, + .cops = &nxt_comp_zstd_ops, + }, +#endif +#if NXT_HAVE_BROTLI + { + .token = nxt_string("br"), + .scheme = NXT_HTTP_COMP_SCHEME_BROTLI, + .def_compr = NXT_HTTP_COMP_BROTLI_DEFAULT_LEVEL, + .cops = &nxt_comp_brotli_ops, + }, +#endif +}; + +static void print_compressor(const nxt_http_comp_compressor_t *c) +{ + printf("token : %s\n", c->type->token.start); + printf("scheme : %d\n", c->type->scheme); + printf("level : %d\n", c->opts.level); + printf("min_len : %ld\n", c->opts.min_len); +} + +static void print_comp_config(size_t n) +{ + for (size_t i = 0; i < n; i++) { + nxt_http_comp_compressor_t *compr = enabled_compressors + i; + + print_compressor(compr); + printf("\n"); + } +} + +nxt_int_t +nxt_http_comp_compress_response(nxt_buf_t *out) +{ + nxt_buf_t *b; + size_t in_len; + size_t buf_len; + ssize_t cbytes; + uint8_t *buf; + bool last = false; + nxt_http_comp_compressor_t *compressor; + + printf("%s: \n", __func__); + + b = out; + + if (b->mem.pos == NULL) { + return NXT_OK; + } + + if (compressor_ctx.idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + printf("%s: NXT_HTTP_COMP_SCHEME_IDENTITY [skipping/identity]\n", + __func__); + return NXT_OK; + } + + compressor = &enabled_compressors[compressor_ctx.idx]; + + in_len = b->mem.free - b->mem.pos; +// last = !b->next || (b->next && b->next->is_last == 1); + if ((b->next && b->next->is_last == 1) + || compressor_ctx.resp_clen == compressor_ctx.clen_procd + + (nxt_off_t)in_len) + { + last = true; + } + + buf_len = compressor->type->cops->compressed_size(&compressor_ctx.ctx, + in_len); + buf = nxt_malloc(buf_len); + cbytes = compressor->type->cops->deflate(&compressor_ctx.ctx, b->mem.pos, + in_len, buf, buf_len, last); + printf("%s: cbytes = %ld\n", __func__, cbytes); + /* XXX What if new buffer is larger than original buffer... ? */ + if (cbytes != -1) { + compressor_ctx.clen_procd += in_len; + b->mem.free = nxt_cpymem(b->mem.pos, buf, cbytes); + } + nxt_free(buf); + + return NXT_OK; +} + +static nxt_uint_t +nxt_http_comp_compressor_lookup_enabled(const nxt_str_t *token) +{ + if (token->start[0] == '*') { + return NXT_HTTP_COMP_SCHEME_IDENTITY; + } + + for (size_t i = 0, n = nr_enabled_compressors; i < n; i++) { + if (nxt_strstr_eq(token, &enabled_compressors[i].type->token)) { + return i; + } + } + + return NXT_HTTP_COMP_SCHEME_UNKNOWN; +} + +/* + * We need to parse the 'Accept-Encoding` header as described by + * + * which can take forms such as + * + * Accept-Encoding: compress, gzip + * Accept-Encoding: + * Accept-Encoding: * + * Accept-Encoding: compress;q=0.5, gzip;q=1.0 + * Accept-Encoding: gzip;q=1.0, identity;q=0.5, *;q=0 + * + * '*:q=0' means if the content being served has no 'Content-Coding' + * matching an 'Accept-Encoding' entry then don't send any response. + * + * 'indentity;q=0' seems to basically mean the same thing... + */ +static nxt_int_t +nxt_http_comp_select_compressor(const nxt_str_t *token) +{ + bool identity_allowed = true; + char *str; + char *saveptr; + double weight = 0.0; + nxt_int_t idx = 0; + + str = strndup((char *)token->start, token->length); + + for ( ; ; str = NULL) { + char *tkn, *qptr; + double qval = -1.0; + nxt_uint_t ecidx; + nxt_str_t enc; + nxt_http_comp_scheme_t scheme; + + tkn = strtok_r(str, ", ", &saveptr); + if (tkn == NULL) { + break; + } + + qptr = strstr(tkn, ";q="); + if (qptr != NULL) { + qval = atof(qptr + 3); + } + + enc.start = (u_char *)tkn; + enc.length = qptr != NULL ? (size_t)(qptr - tkn) : strlen(tkn); + + ecidx = nxt_http_comp_compressor_lookup_enabled(&enc); + if (ecidx == NXT_HTTP_COMP_SCHEME_UNKNOWN) { + continue; + } + + scheme = enabled_compressors[ecidx].type->scheme; + + printf("%s: %.*s [%f] [%d:%d]\n", __func__, (int)enc.length, enc.start, + qval, ecidx, scheme); + + if (qval == 0.0 && scheme == NXT_HTTP_COMP_SCHEME_IDENTITY) { + identity_allowed = false; + } + + if (qval != -1.0 && (qval == 0.0 || qval < weight)) { + continue; + } + + idx = ecidx; + weight = qval; + } + + free(str); + + printf("%s: Selected compressor : %s\n", __func__, + enabled_compressors[idx].type->token.start); + + printf("%s: idx [%u], identity_allowed [%s]\n", __func__, idx, + identity_allowed ? "true" : "false"); + + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY && !identity_allowed) { + return -1; + } + + return idx; +} + +static nxt_int_t +nxt_http_comp_set_header(nxt_http_request_t *r, nxt_uint_t index) +{ + const nxt_str_t *token; + nxt_http_field_t *f; + + static const nxt_str_t content_encoding_str = + nxt_string("Content-Encoding"); + + f = nxt_list_add(r->resp.fields); + if (nxt_slow_path(f == NULL)) { + return NXT_ERROR; + } + + r->resp.content_length = NULL; + r->resp.content_length_n = -1; + + f->hash = 0; + f->skip = 0; + f->hopbyhop = 0; + + token = &enabled_compressors[index].type->token; + + f->name = content_encoding_str.start; + f->name_length = content_encoding_str.length; + f->value = token->start; + f->value_length = token->length; + + return NXT_OK; +} + +static bool +nxt_http_comp_is_resp_content_encoded(const nxt_http_request_t *r) +{ + nxt_http_field_t *f; + + printf("%s: \n", __func__); + + nxt_list_each(f, r->resp.fields) { + printf("%s: %s: %s\n", __func__, f->name, f->value); + if (nxt_strcasecmp(f->name, (const u_char *)"Content-Encoding") == 0) { + return true; + } + } nxt_list_loop; + + return false; +} + +nxt_int_t +nxt_http_comp_check_compression(nxt_task_t *task, nxt_http_request_t *r) +{ + int8_t level; + nxt_str_t mime_type = { }; + nxt_int_t ret, idx; + nxt_off_t min_len; + nxt_str_t accept_encoding; + nxt_router_conf_t *rtcf; + nxt_http_comp_compressor_t *compressor; + + printf("%s: \n", __func__); + + compressor_ctx = (nxt_http_comp_ctx_t){ .resp_clen = -1 }; + + if (r->resp.content_length_n == 0) { + return NXT_OK; + } + + if (r->resp.mime_type != NULL) { + mime_type = *r->resp.mime_type; + } else if (r->resp.content_type != NULL) { + mime_type.start = r->resp.content_type->value; + mime_type.length = r->resp.content_type->value_length; + } + + if (mime_type.start == NULL) { + return NXT_OK; + } + + printf("%s: Response Content-Type [%.*s]\n", __func__, + (int)mime_type.length, mime_type.start); + + if (mime_types_rule != NULL) { + ret = nxt_http_route_test_rule(r, mime_types_rule, + mime_type.start, + mime_type.length); + printf("%s: mime_type : %d (%.*s)\n", __func__, ret, + (int)mime_type.length, mime_type.start); + + if (ret == 0) { + return NXT_OK; + } + } + + rtcf = r->conf->socket_conf->router_conf; + + if (nxt_http_comp_is_resp_content_encoded(r)) { + return NXT_OK; + } + + /* XXX Should checking the Accept-Encoding header come first? */ + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache, + r, r->mem_pool); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + + nxt_tstr_query(task, r->tstr_query, accept_encoding_query, + &accept_encoding); + if (nxt_slow_path(nxt_tstr_query_failed(r->tstr_query))) { + return NXT_ERROR; + } + + printf("%s: Accept-Encoding: %s\n", __func__, accept_encoding.start); + + idx = nxt_http_comp_select_compressor(&accept_encoding); + if (idx == -1) { + /* + * XXX Needs to trigger the right HTTP error status code + * for unsuppported media or something... + */ + return NXT_ERROR; + } + + if (idx == NXT_HTTP_COMP_SCHEME_IDENTITY) { + return NXT_OK; + } + + compressor = &enabled_compressors[idx]; + + if (r->resp.content_length_n > -1) { + compressor_ctx.resp_clen = r->resp.content_length_n; + } else if (r->resp.content_length != NULL) { + compressor_ctx.resp_clen = + strtol((char *)r->resp.content_length->value, NULL, 10); + } + + min_len = compressor->opts.min_len; + + if (compressor_ctx.resp_clen > -1 && compressor_ctx.resp_clen < min_len) { + printf("%s: %ld < %ld [skipping/clen]\n", __func__, + compressor_ctx.resp_clen , min_len); + return NXT_OK; + } + + nxt_http_comp_set_header(r, idx); + + compressor_ctx.idx = idx; + + level = enabled_compressors[idx].opts.level; + compressor_ctx.ctx.level = level == NXT_COMP_LEVEL_UNSET ? + enabled_compressors[idx].type->def_compr : level; + + compressor->type->cops->init(&compressor_ctx.ctx); + + return NXT_OK; +} + +static nxt_uint_t +nxt_http_comp_compressor_token2idx(const nxt_str_t *token) +{ + for (nxt_uint_t i = 0, n = nxt_nitems(compressors); i < n; i++) { + if (nxt_strstr_eq(token, &compressors[i].token)) { + return i; + } + } + + return NXT_HTTP_COMP_SCHEME_UNKNOWN; +} + +bool +nxt_http_comp_compressor_is_valid(const nxt_str_t *token) +{ + nxt_uint_t idx; + + idx = nxt_http_comp_compressor_token2idx(token); + if (idx != NXT_HTTP_COMP_SCHEME_UNKNOWN) { + return true; + } + + return false; +} + +static nxt_int_t +nxt_http_comp_set_compressor(nxt_router_conf_t *rtcf, + const nxt_conf_value_t *comp, nxt_uint_t index) +{ + nxt_int_t ret; + nxt_str_t token; + nxt_uint_t cidx; + nxt_conf_value_t *obj; + + printf("%s: \n", __func__); + + static nxt_str_t token_str = nxt_string("encoding"); + + obj = nxt_conf_get_object_member(comp, &token_str, NULL); + if (obj == NULL) { + return NXT_ERROR; + } + + nxt_conf_get_string(obj, &token); + cidx = nxt_http_comp_compressor_token2idx(&token); + + enabled_compressors[index].type = &compressors[cidx]; + enabled_compressors[index].opts.level = NXT_COMP_LEVEL_UNSET; + enabled_compressors[index].opts.min_len = -1; + printf("%s: %s\n", __func__, enabled_compressors[index].type->token.start); + + ret = nxt_conf_map_object(rtcf->mem_pool, comp, compressors_opts_map, + nxt_nitems(compressors_opts_map), + &enabled_compressors[index].opts); + if (nxt_slow_path(ret == NXT_ERROR)) { + return NXT_ERROR; + } + + return NXT_OK; +} + +nxt_int_t +nxt_http_comp_compression_init(nxt_task_t *task, nxt_router_conf_t *rtcf, + const nxt_conf_value_t *comp_conf) +{ + nxt_int_t ret; + nxt_uint_t n = 1; /* 'identity' */ + nxt_conf_value_t *comps, *mimes; + + static const nxt_str_t accept_enc_str = + nxt_string("$header_accept_encoding"); + static const nxt_str_t comps_str = nxt_string("compressors"); + static const nxt_str_t mimes_str = nxt_string("types"); + + printf("%s: \n", __func__); + + mimes = nxt_conf_get_object_member(comp_conf, &mimes_str, NULL); + if (mimes != NULL) { + mime_types_rule = nxt_http_route_types_rule_create(task, + rtcf->mem_pool, + mimes); + if (nxt_slow_path(mime_types_rule == NULL)) { + return NXT_ERROR; + } + } + + accept_encoding_query = nxt_tstr_compile(rtcf->tstr_state, &accept_enc_str, + NXT_TSTR_STRZ); + if (nxt_slow_path(accept_encoding_query == NULL)) { + return NXT_ERROR; + } + + comps = nxt_conf_get_object_member(comp_conf, &comps_str, NULL); + if (nxt_slow_path(comps == NULL)) { + return NXT_ERROR; + } + + if (nxt_conf_type(comps) == NXT_CONF_OBJECT) { + n++; + } else { + n += nxt_conf_object_members_count(comps); + } + nr_enabled_compressors = n; + + enabled_compressors = nxt_mp_zalloc(rtcf->mem_pool, + sizeof(nxt_http_comp_compressor_t) * n); + + enabled_compressors[0] = + (nxt_http_comp_compressor_t){ .type = &compressors[0], + .opts.level = NXT_COMP_LEVEL_UNSET, + .opts.min_len = -1 }; + + if (nxt_conf_type(comps) == NXT_CONF_OBJECT) { + /* XXX Remove me... */ + print_comp_config(nr_enabled_compressors); + return nxt_http_comp_set_compressor(rtcf, comps, 1); + } + + for (nxt_uint_t i = 1; i < n; i++) { + nxt_conf_value_t *obj; + + obj = nxt_conf_get_array_element(comps, i - 1); + ret = nxt_http_comp_set_compressor(rtcf, obj, i); + if (ret == NXT_ERROR) { + return NXT_ERROR; + } + } + + nr_enabled_compressors = n; + /* XXX Remove me... */ + print_comp_config(nr_enabled_compressors); + + return NXT_OK; +} diff --git a/src/nxt_http_compression.h b/src/nxt_http_compression.h new file mode 100644 index 00000000..a76f3993 --- /dev/null +++ b/src/nxt_http_compression.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) Andrew Clayton + * Copyright (C) F5, Inc. + */ + +#ifndef _NXT_COMPRESSION_H_INCLUDED_ +#define _NXT_COMPRESSION_H_INCLUDED_ + +#include + +#include +#include +#include + +#if NXT_HAVE_ZLIB +#include +#endif + +#if NXT_HAVE_ZSTD +#include +#endif + +#if NXT_HAVE_BROTLI +#include +#endif + +#include +#include +#include +#include + +#if NXT_HAVE_ZLIB +#define NXT_HTTP_COMP_ZLIB_DEFAULT_LEVEL Z_DEFAULT_COMPRESSION +#endif +#if NXT_HAVE_ZSTD +#define NXT_HTTP_COMP_ZSTD_DEFAULT_LEVEL ZSTD_CLEVEL_DEFAULT +#endif +#if NXT_HAVE_BROTLI +#define NXT_HTTP_COMP_BROTLI_DEFAULT_LEVEL BROTLI_DEFAULT_QUALITY +#endif + +typedef struct nxt_http_comp_compressor_ctx_s nxt_http_comp_compressor_ctx_t; +typedef struct nxt_http_comp_operations_s nxt_http_comp_operations_t; + +struct nxt_http_comp_compressor_ctx_s { + int8_t level; + + union { +#if NXT_HAVE_ZLIB + z_stream zlib_ctx; +#endif +#if NXT_HAVE_ZSTD + ZSTD_CStream *zstd_ctx; +#endif +#if NXT_HAVE_BROTLI + BrotliEncoderState *brotli_ctx; +#endif + }; +}; + +struct nxt_http_comp_operations_s { + void (*init)(nxt_http_comp_compressor_ctx_t *ctx); + size_t (*compressed_size)(const nxt_http_comp_compressor_ctx_t *ctx, + size_t in_len); + ssize_t (*deflate)(nxt_http_comp_compressor_ctx_t *ctx, + const uint8_t *in_buf, size_t in_len, + uint8_t *out_buf, size_t out_len, bool last); + void (*free_ctx)(const nxt_http_comp_compressor_ctx_t *ctx); +}; + +#if NXT_HAVE_ZLIB +extern const nxt_http_comp_operations_t nxt_comp_deflate_ops; +extern const nxt_http_comp_operations_t nxt_comp_gzip_ops; +#endif + +#if NXT_HAVE_ZSTD +extern const nxt_http_comp_operations_t nxt_comp_zstd_ops; +#endif + +#if NXT_HAVE_BROTLI +extern const nxt_http_comp_operations_t nxt_comp_brotli_ops; +#endif + +extern bool nxt_http_comp_compressor_is_valid(const nxt_str_t *token); +extern nxt_int_t nxt_http_comp_check_compression(nxt_task_t *task, + nxt_http_request_t *r); +extern nxt_int_t nxt_http_comp_compress_response(nxt_buf_t *out); +extern nxt_int_t nxt_http_comp_compression_init(nxt_task_t *task, + nxt_router_conf_t *rtcf, const nxt_conf_value_t *comp_conf); + +#endif /* _NXT_COMPRESSION_H_INCLUDED_ */ diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 425a4607..0a89b327 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -6,6 +6,7 @@ #include #include +#include static nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp); @@ -311,6 +312,8 @@ nxt_http_request_start(nxt_task_t *task, void *obj, void *data) nxt_socket_conf_t *skcf; nxt_http_request_t *r; + printf("%s: \n", __func__); + r = obj; r->state = &nxt_http_request_body_state; @@ -636,6 +639,13 @@ nxt_http_request_header_send(nxt_task_t *task, nxt_http_request_t *r, nxt_http_field_t *server, *date, *content_length; nxt_socket_conf_t *skcf; + printf("%s: \n", __func__); + + ret = nxt_http_comp_check_compression(task, r); + if (nxt_slow_path(ret != NXT_OK)) { + goto fail; + } + ret = nxt_http_set_headers(r); if (nxt_slow_path(ret != NXT_OK)) { goto fail; @@ -726,6 +736,10 @@ nxt_http_request_ws_frame_start(nxt_task_t *task, nxt_http_request_t *r, void nxt_http_request_send(nxt_task_t *task, nxt_http_request_t *r, nxt_buf_t *out) { + printf("%s: \n", __func__); + + nxt_http_comp_compress_response(out); + if (nxt_fast_path(r->proto.any != NULL)) { nxt_http_proto[r->protocol].send(task, r, out); } diff --git a/src/nxt_http_route.c b/src/nxt_http_route.c index 852f5739..0e0e8e7a 100644 --- a/src/nxt_http_route.c +++ b/src/nxt_http_route.c @@ -656,6 +656,8 @@ nxt_http_action_init(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_router_conf_t *rtcf; nxt_http_action_conf_t acf; + printf("%s: \n", __func__); + nxt_memzero(&acf, sizeof(acf)); ret = nxt_conf_map_object(tmcf->mem_pool, cv, nxt_http_route_action_conf, diff --git a/src/nxt_http_static.c b/src/nxt_http_static.c index ee25015e..c8828782 100644 --- a/src/nxt_http_static.c +++ b/src/nxt_http_static.c @@ -647,6 +647,8 @@ nxt_http_static_send_ready(nxt_task_t *task, void *obj, void *data) body_handler = NULL; } + r->resp.mime_type = mtype; + nxt_http_request_header_send(task, r, body_handler, NULL); r->state = &nxt_http_static_send_state; @@ -799,6 +801,8 @@ nxt_http_static_body_handler(nxt_task_t *task, void *obj, void *data) nxt_work_queue_t *wq; nxt_http_request_t *r; + printf("%s: \n", __func__); + r = obj; fb = r->out; @@ -859,6 +863,8 @@ nxt_http_static_buf_completion(nxt_task_t *task, void *obj, void *data) nxt_off_t rest; nxt_http_request_t *r; + printf("%s: \n", __func__); + b = obj; r = data; diff --git a/src/nxt_router.c b/src/nxt_router.c index 48870d20..50ec44b5 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -21,6 +21,7 @@ #include #include #include +#include #define NXT_SHARED_PORT_ID 0xFFFFu @@ -1655,6 +1656,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, static const nxt_str_t static_path = nxt_string("/settings/http/static"); static const nxt_str_t websocket_path = nxt_string("/settings/http/websocket"); + static const nxt_str_t compression_path = + nxt_string("/settings/http/compression"); static const nxt_str_t forwarded_path = nxt_string("/forwarded"); static const nxt_str_t client_ip_path = nxt_string("/client_ip"); @@ -2002,6 +2005,8 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_str_null(&skcf->body_temp_path); if (http != NULL) { + nxt_conf_value_t *comp; + ret = nxt_conf_map_object(mp, http, nxt_router_http_conf, nxt_nitems(nxt_router_http_conf), skcf); @@ -2009,6 +2014,11 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, nxt_alert(task, "http map error"); goto fail; } + + comp = nxt_conf_get_path(root, &compression_path); + if (comp != NULL) { + nxt_http_comp_compression_init(task, rtcf, comp); + } } if (websocket != NULL) { diff --git a/src/nxt_zlib.c b/src/nxt_zlib.c new file mode 100644 index 00000000..2dcc53a8 --- /dev/null +++ b/src/nxt_zlib.c @@ -0,0 +1,79 @@ +/* + * + */ + +#include +#include +#include + +#include + +#include + +static void nxt_zlib_gzip_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + int ret; + z_stream *z = &ctx->zlib_ctx; + + *z = (z_stream){ }; + + ret = deflateInit2(z, ctx->level, Z_DEFLATED, 9 + 16, 8, + Z_DEFAULT_STRATEGY); +} + +static void nxt_zlib_deflate_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + int ret; + z_stream *z = &ctx->zlib_ctx; + + *z = (z_stream){ }; + + ret = deflateInit2(z, ctx->level, Z_DEFLATED, 9, 8, Z_DEFAULT_STRATEGY); +} + +static size_t nxt_zlib_compressed_size(const nxt_http_comp_compressor_ctx_t *ctx, + size_t in_len) +{ + z_stream *z = &ctx->zlib_ctx; + + return deflateBound(z, in_len); +} + +static ssize_t nxt_zlib_deflate(nxt_http_comp_compressor_ctx_t *ctx, + const uint8_t *in_buf, size_t in_len, + uint8_t *out_buf, size_t out_len, bool last) +{ + int ret; + z_stream *z = &ctx->zlib_ctx; + size_t compressed_bytes = z->total_out; + + z->avail_in = in_len; + z->next_in = (z_const Bytef *)in_buf; + + z->avail_out = out_len; + z->next_out = out_buf; + + ret = deflate(z, last ? Z_FINISH : Z_SYNC_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_BUF_ERROR) { + deflateEnd(z); + printf("%s: ret = %d\n", __func__, ret); + return -1; + } + + if (last) + deflateEnd(z); + + return z->total_out - compressed_bytes; +} + +const nxt_http_comp_operations_t nxt_comp_deflate_ops = { + .init = nxt_zlib_deflate_init, + .compressed_size = nxt_zlib_compressed_size, + .deflate = nxt_zlib_deflate, +}; + +const nxt_http_comp_operations_t nxt_comp_gzip_ops = { + .init = nxt_zlib_gzip_init, + .compressed_size = nxt_zlib_compressed_size, + .deflate = nxt_zlib_deflate, +}; diff --git a/src/nxt_zstd.c b/src/nxt_zstd.c new file mode 100644 index 00000000..07da9ef6 --- /dev/null +++ b/src/nxt_zstd.c @@ -0,0 +1,73 @@ +/* + * + */ + +#include +#include +#include + +#include + +#include + +static void nxt_zstd_free(const nxt_http_comp_compressor_ctx_t *ctx) +{ + ZSTD_CStream *zstd = ctx->zstd_ctx; + + ZSTD_freeCStream(zstd); +} + +static void nxt_zstd_init(nxt_http_comp_compressor_ctx_t *ctx) +{ + ZSTD_CStream **zstd = &ctx->zstd_ctx; + + *zstd = ZSTD_createCStream(); + ZSTD_initCStream(*zstd, ctx->level); + + printf("%s: zstd compression level [%d]\n", __func__, ctx->level); +} + +static size_t nxt_zstd_compressed_size(const nxt_http_comp_compressor_ctx_t *ctx, + size_t in_len) +{ + return ZSTD_compressBound(in_len); +} + +static ssize_t nxt_zstd_compress(nxt_http_comp_compressor_ctx_t *ctx, + const uint8_t *in_buf, size_t in_len, + uint8_t *out_buf, size_t out_len, bool last) +{ + size_t ret; + ZSTD_CStream *zstd = ctx->zstd_ctx; + ZSTD_inBuffer zinb = { .src = in_buf, .size = in_len }; + ZSTD_outBuffer zoutb = { .dst = out_buf, .size = out_len }; + + printf("%s: in_len [%lu] out_len [%lu]\n", __func__, in_len, out_len); + + ret = ZSTD_compressStream(zstd, &zoutb, &zinb); + + if (zinb.pos < zinb.size) { + printf("%s: short by [%d]\n", __func__, zinb.pos < zinb.size); + ret = ZSTD_flushStream(zstd, &zoutb); + } + + if (last) { + ret = ZSTD_endStream(zstd, &zoutb); + nxt_zstd_free(ctx); + } + + printf("%s: ret [%lu]\n", __func__, ret); + if (ZSTD_isError(ret)) { + printf("%s: [%s]\n", __func__, ZSTD_getErrorName(ret)); + return -1; + } + + return zoutb.pos; +} + +const nxt_http_comp_operations_t nxt_comp_zstd_ops = { + .init = nxt_zstd_init, + .compressed_size = nxt_zstd_compressed_size, + .deflate = nxt_zstd_compress, + .free_ctx = nxt_zstd_free, +}; -- cgit