From 76cc071ab16cf342afb79b5e0976558215785f2f Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 14 Oct 2024 15:16:40 +0800 Subject: Fix missing newlines in access logs for JS configuration When using JS configuration for the "format" option, access log entries were being written without newline characters. This commit adds the missing newline character to each log entry. Closes: https://github.com/nginx/unit/issues/1458 --- src/nxt_router_access_log.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) (limited to 'src/nxt_router_access_log.c') diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index afecd0b1..024bff39 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -65,7 +65,6 @@ nxt_int_t nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *value) { - u_char *p; nxt_int_t ret; nxt_str_t str; nxt_tstr_t *format; @@ -121,18 +120,8 @@ nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_memcpy(access_log->path.start, alcf.path.start, alcf.path.length); } - str.length = alcf.format.length + 1; - - str.start = nxt_malloc(str.length); - if (str.start == NULL) { - nxt_alert(task, "failed to allocate log format structure"); - return NXT_ERROR; - } - - p = nxt_cpymem(str.start, alcf.format.start, alcf.format.length); - *p = '\n'; - - format = nxt_tstr_compile(rtcf->tstr_state, &str, NXT_TSTR_LOGGING); + format = nxt_tstr_compile(rtcf->tstr_state, &alcf.format, + NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); if (nxt_slow_path(format == NULL)) { return NXT_ERROR; } -- cgit From a92d8149e47334d23e8cd31c48e3f02030049e30 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 28 Oct 2024 15:22:27 +0800 Subject: http: Refactor format field in nxt_router_access_log_conf_t This is a preparatory refactoring for upcoming JSON format support in access log. No functional changes. --- src/nxt_router_access_log.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'src/nxt_router_access_log.c') diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index 024bff39..cc34fd94 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -12,7 +12,7 @@ typedef struct { nxt_str_t path; - nxt_str_t format; + nxt_conf_value_t *format; nxt_conf_value_t *expr; } nxt_router_access_log_conf_t; @@ -49,7 +49,7 @@ static nxt_conf_map_t nxt_router_access_log_conf[] = { { nxt_string("format"), - NXT_CONF_MAP_STR, + NXT_CONF_MAP_PTR, offsetof(nxt_router_access_log_conf_t, format), }, @@ -67,19 +67,16 @@ nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, { nxt_int_t ret; nxt_str_t str; - nxt_tstr_t *format; nxt_router_t *router; nxt_router_access_log_t *access_log; nxt_router_access_log_conf_t alcf; - static const nxt_str_t log_format_str = nxt_string("$remote_addr - - " + static const nxt_str_t default_format = nxt_string("$remote_addr - - " "[$time_local] \"$request_line\" $status $body_bytes_sent " "\"$header_referer\" \"$header_user_agent\""); nxt_memzero(&alcf, sizeof(nxt_router_access_log_conf_t)); - alcf.format = log_format_str; - if (nxt_conf_type(value) == NXT_CONF_STRING) { nxt_conf_get_string(value, &alcf.path); @@ -120,14 +117,20 @@ nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_memcpy(access_log->path.start, alcf.path.start, alcf.path.length); } - format = nxt_tstr_compile(rtcf->tstr_state, &alcf.format, - NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); - if (nxt_slow_path(format == NULL)) { - return NXT_ERROR; + rtcf->access_log = access_log; + + if (alcf.format != NULL) { + nxt_conf_get_string(alcf.format, &str); + + } else { + str = default_format; } - rtcf->access_log = access_log; - rtcf->log_format = format; + rtcf->log_format = nxt_tstr_compile(rtcf->tstr_state, &str, + NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); + if (nxt_slow_path(rtcf->log_format == NULL)) { + return NXT_ERROR; + } if (alcf.expr != NULL) { nxt_conf_get_string(alcf.expr, &str); -- cgit From a760e24a2c4e5f93703bb365b12807d8b1035cf5 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 28 Oct 2024 16:30:03 +0800 Subject: http: Introduce nxt_router_access_log_format_t structure This is a preparatory refactoring for upcoming JSON format support in access log. We will extend format option to access object for JSON support. No functional changes. --- src/nxt_router_access_log.c | 66 ++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 18 deletions(-) (limited to 'src/nxt_router_access_log.c') diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index cc34fd94..30b38aa1 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -23,9 +23,16 @@ typedef struct { } nxt_router_access_log_ctx_t; +struct nxt_router_access_log_format_s { + nxt_tstr_t *tstr; +}; + + +static nxt_router_access_log_format_t *nxt_router_access_log_format_create( + nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *value); static void nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, nxt_router_access_log_t *access_log, - nxt_tstr_t *format); + nxt_router_access_log_format_t *format); static void nxt_router_access_log_write(nxt_task_t *task, nxt_http_request_t *r, nxt_router_access_log_ctx_t *ctx); static void nxt_router_access_log_ready(nxt_task_t *task, @@ -71,10 +78,6 @@ nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_router_access_log_t *access_log; nxt_router_access_log_conf_t alcf; - static const nxt_str_t default_format = nxt_string("$remote_addr - - " - "[$time_local] \"$request_line\" $status $body_bytes_sent " - "\"$header_referer\" \"$header_user_agent\""); - nxt_memzero(&alcf, sizeof(nxt_router_access_log_conf_t)); if (nxt_conf_type(value) == NXT_CONF_STRING) { @@ -119,15 +122,8 @@ nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, rtcf->access_log = access_log; - if (alcf.format != NULL) { - nxt_conf_get_string(alcf.format, &str); - - } else { - str = default_format; - } - - rtcf->log_format = nxt_tstr_compile(rtcf->tstr_state, &str, - NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); + rtcf->log_format = nxt_router_access_log_format_create(task, rtcf, + alcf.format); if (nxt_slow_path(rtcf->log_format == NULL)) { return NXT_ERROR; } @@ -145,9 +141,43 @@ nxt_router_access_log_create(nxt_task_t *task, nxt_router_conf_t *rtcf, } +static nxt_router_access_log_format_t * +nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, + nxt_conf_value_t *value) +{ + nxt_str_t str; + nxt_router_access_log_format_t *format; + + static const nxt_str_t default_format = nxt_string("$remote_addr - - " + "[$time_local] \"$request_line\" $status $body_bytes_sent " + "\"$header_referer\" \"$header_user_agent\""); + + format = nxt_mp_zalloc(rtcf->mem_pool, + sizeof(nxt_router_access_log_format_t)); + if (nxt_slow_path(format == NULL)) { + return NULL; + } + + if (value != NULL) { + nxt_conf_get_string(value, &str); + + } else { + str = default_format; + } + + format->tstr = nxt_tstr_compile(rtcf->tstr_state, &str, + NXT_TSTR_LOGGING | NXT_TSTR_NEWLINE); + if (nxt_slow_path(format->tstr == NULL)) { + return NULL; + } + + return format; +} + + static void nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, - nxt_router_access_log_t *access_log, nxt_tstr_t *format) + nxt_router_access_log_t *access_log, nxt_router_access_log_format_t *format) { nxt_int_t ret; nxt_router_conf_t *rtcf; @@ -160,8 +190,8 @@ nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, ctx->access_log = access_log; - if (nxt_tstr_is_const(format)) { - nxt_tstr_str(format, &ctx->text); + if (nxt_tstr_is_const(format->tstr)) { + nxt_tstr_str(format->tstr, &ctx->text); } else { rtcf = r->conf->socket_conf->router_conf; @@ -172,7 +202,7 @@ nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, return; } - ret = nxt_tstr_query(task, r->tstr_query, format, &ctx->text); + ret = nxt_tstr_query(task, r->tstr_query, format->tstr, &ctx->text); if (nxt_slow_path(ret != NXT_OK)) { return; } -- cgit From bc838c5e1b6270c5195c4bdaa6a0a4bb72e48549 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Mon, 28 Oct 2024 17:45:19 +0800 Subject: http: Support JSON format in access log Allow format to be an object to generate JSON logs. The object keys become JSON field names, and values support string, variable, and JS. Note that when there is no JS in the format values, the object will be pre-serialized to a JSON template string at configuration phase for better performance. Example config: { "access_log": { "path": "/tmp/access.log", "format": { "remote_addr": "$remote_addr", "time_local": "$time_local", "request_line": "$request_line", "status": "$status", "body_bytes_sent": "$body_bytes_sent", "header_referer": "$header_referer", "header_user_agent": "$header_user_agent" } } } --- src/nxt_router_access_log.c | 194 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 179 insertions(+), 15 deletions(-) (limited to 'src/nxt_router_access_log.c') diff --git a/src/nxt_router_access_log.c b/src/nxt_router_access_log.c index 30b38aa1..1ec1a2d2 100644 --- a/src/nxt_router_access_log.c +++ b/src/nxt_router_access_log.c @@ -11,20 +11,28 @@ typedef struct { - nxt_str_t path; - nxt_conf_value_t *format; - nxt_conf_value_t *expr; + nxt_str_t path; + nxt_conf_value_t *format; + nxt_conf_value_t *expr; } nxt_router_access_log_conf_t; typedef struct { - nxt_str_t text; - nxt_router_access_log_t *access_log; + nxt_str_t text; + nxt_router_access_log_t *access_log; } nxt_router_access_log_ctx_t; +typedef struct { + nxt_str_t name; + nxt_tstr_t *tstr; +} nxt_router_access_log_member_t; + + struct nxt_router_access_log_format_s { - nxt_tstr_t *tstr; + nxt_tstr_t *tstr; + nxt_uint_t nmembers; + nxt_router_access_log_member_t *member; }; @@ -33,6 +41,11 @@ static nxt_router_access_log_format_t *nxt_router_access_log_format_create( static void nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, nxt_router_access_log_t *access_log, nxt_router_access_log_format_t *format); +static nxt_int_t nxt_router_access_log_text(nxt_task_t *task, + nxt_http_request_t *r, nxt_router_access_log_ctx_t *ctx, nxt_tstr_t *tstr); +static nxt_int_t nxt_router_access_log_json(nxt_task_t *task, + nxt_http_request_t *r, nxt_router_access_log_ctx_t *ctx, + nxt_router_access_log_format_t *format); static void nxt_router_access_log_write(nxt_task_t *task, nxt_http_request_t *r, nxt_router_access_log_ctx_t *ctx); static void nxt_router_access_log_ready(nxt_task_t *task, @@ -145,7 +158,12 @@ static nxt_router_access_log_format_t * nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, nxt_conf_value_t *value) { - nxt_str_t str; + size_t size; + uint32_t i, n, next; + nxt_str_t name, str, *dst; + nxt_bool_t has_js; + nxt_conf_value_t *cv; + nxt_router_access_log_member_t *member; nxt_router_access_log_format_t *format; static const nxt_str_t default_format = nxt_string("$remote_addr - - " @@ -159,7 +177,74 @@ nxt_router_access_log_format_create(nxt_task_t *task, nxt_router_conf_t *rtcf, } if (value != NULL) { - nxt_conf_get_string(value, &str); + + if (nxt_conf_type(value) == NXT_CONF_OBJECT) { + next = 0; + has_js = 0; + + n = nxt_conf_object_members_count(value); + + for ( ;; ) { + cv = nxt_conf_next_object_member(value, &name, &next); + if (cv == NULL) { + break; + } + + nxt_conf_get_string(cv, &str); + + if (nxt_tstr_is_js(&str)) { + has_js = 1; + } + } + + if (has_js) { + member = nxt_mp_alloc(rtcf->mem_pool, + n * sizeof(nxt_router_access_log_member_t)); + if (nxt_slow_path(member == NULL)) { + return NULL; + } + + next = 0; + + for (i = 0; i < n; i++) { + cv = nxt_conf_next_object_member(value, &name, &next); + if (cv == NULL) { + break; + } + + dst = nxt_str_dup(rtcf->mem_pool, &member[i].name, &name); + if (nxt_slow_path(dst == NULL)) { + return NULL; + } + + nxt_conf_get_string(cv, &str); + + member[i].tstr = nxt_tstr_compile(rtcf->tstr_state, &str, + NXT_TSTR_LOGGING); + if (nxt_slow_path(member[i].tstr == NULL)) { + return NULL; + } + } + + format->nmembers = n; + format->member = member; + + return format; + } + + size = nxt_conf_json_length(value, NULL); + + str.start = nxt_mp_nget(rtcf->mem_pool, size); + if (nxt_slow_path(str.start == NULL)) { + return NULL; + } + + str.length = nxt_conf_json_print(str.start, value, NULL) + - str.start; + + } else { + nxt_conf_get_string(value, &str); + } } else { str = default_format; @@ -180,7 +265,6 @@ nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, nxt_router_access_log_t *access_log, nxt_router_access_log_format_t *format) { nxt_int_t ret; - nxt_router_conf_t *rtcf; nxt_router_access_log_ctx_t *ctx; ctx = nxt_mp_get(r->mem_pool, sizeof(nxt_router_access_log_ctx_t)); @@ -190,8 +274,28 @@ nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, ctx->access_log = access_log; - if (nxt_tstr_is_const(format->tstr)) { - nxt_tstr_str(format->tstr, &ctx->text); + if (format->tstr != NULL) { + ret = nxt_router_access_log_text(task, r, ctx, format->tstr); + + } else { + ret = nxt_router_access_log_json(task, r, ctx, format); + } + + if (ret == NXT_OK) { + nxt_router_access_log_write(task, r, ctx); + } +} + + +static nxt_int_t +nxt_router_access_log_text(nxt_task_t *task, nxt_http_request_t *r, + nxt_router_access_log_ctx_t *ctx, nxt_tstr_t *tstr) +{ + nxt_int_t ret; + nxt_router_conf_t *rtcf; + + if (nxt_tstr_is_const(tstr)) { + nxt_tstr_str(tstr, &ctx->text); } else { rtcf = r->conf->socket_conf->router_conf; @@ -199,16 +303,76 @@ nxt_router_access_log_writer(nxt_task_t *task, nxt_http_request_t *r, ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, &r->tstr_cache, r, r->mem_pool); if (nxt_slow_path(ret != NXT_OK)) { - return; + return NXT_ERROR; } - ret = nxt_tstr_query(task, r->tstr_query, format->tstr, &ctx->text); + ret = nxt_tstr_query(task, r->tstr_query, tstr, &ctx->text); if (nxt_slow_path(ret != NXT_OK)) { - return; + return NXT_ERROR; } } - nxt_router_access_log_write(task, r, ctx); + return NXT_OK; +} + + +static nxt_int_t +nxt_router_access_log_json(nxt_task_t *task, nxt_http_request_t *r, + nxt_router_access_log_ctx_t *ctx, nxt_router_access_log_format_t *format) +{ + u_char *p; + size_t size; + nxt_int_t ret; + nxt_str_t str; + nxt_uint_t i; + nxt_conf_value_t *value; + nxt_router_conf_t *rtcf; + nxt_router_access_log_member_t *member; + + rtcf = r->conf->socket_conf->router_conf; + + value = nxt_conf_create_object(r->mem_pool, format->nmembers); + if (nxt_slow_path(value == NULL)) { + return NXT_ERROR; + } + + for (i = 0; i < format->nmembers; i++) { + member = &format->member[i]; + + if (nxt_tstr_is_const(member->tstr)) { + nxt_tstr_str(member->tstr, &str); + + } else { + ret = nxt_tstr_query_init(&r->tstr_query, rtcf->tstr_state, + &r->tstr_cache, r, r->mem_pool); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + + ret = nxt_tstr_query(task, r->tstr_query, member->tstr, &str); + if (nxt_slow_path(ret != NXT_OK)) { + return NXT_ERROR; + } + } + + nxt_conf_set_member_string(value, &member->name, &str, i); + } + + size = nxt_conf_json_length(value, NULL) + 1; + + p = nxt_mp_nget(r->mem_pool, size); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + ctx->text.start = p; + + p = nxt_conf_json_print(p, value, NULL); + *p++ = '\n'; + + ctx->text.length = p - ctx->text.start; + + return NXT_OK; } -- cgit