otel: add build tooling to include otel code

Adds the --otel flag to the configure command and the various build time
variables and checks that are needed in this flow.

It also includes the nxt_otel.c and nxt_otel.h files that are needed for
the rest of Unit to talk to the compiled static library that's generated
from the rust crate.

Signed-off-by: Ava Hahn <a.hahn@f5.com>
Co-authored-by: Gabor Javorszky <g.javorszky@f5.com>
Signed-off-by: Gabor Javorszky <g.javorszky@f5.com>

12 files changed, 621 insertions, 11 deletions

diff --git a/auto/help b/auto/help
index 6a6aee19..94854762 100644
--- a/auto/help
+++ b/auto/help
@@ -52,6 +52,8 @@ cat << END
 
   --njs                enable njs library usage
 
+  --otel               enable otel library usage
+
   --debug              enable debug logging
 
   --fuzz=ENGINE        enable fuzz testing
diff --git a/auto/make b/auto/make
index f21a2dfc..183e95c7 100644
--- a/auto/make
+++ b/auto/make
@@ -7,7 +7,6 @@
 
 $echo "creating $NXT_MAKEFILE"
 
-
 cat << END > $NXT_MAKEFILE
 
 # Pretty print compiler etc actions...
@@ -22,6 +21,8 @@ AR =			$AR
 
 EXTRA_CFLAGS =
 CFLAGS = 		$NXT_CFLAGS $NXT_CC_OPT $CFLAGS \$(EXTRA_CFLAGS)
+RUST_FLAGS =
+NXT_OTEL_LIB_LOC =
 
 NXT_EXEC_LINK =		$NXT_EXEC_LINK $NXT_LD_OPT
 NXT_SHARED_LOCAL_LINK =	$NXT_SHARED_LOCAL_LINK $NXT_LD_OPT
@@ -62,6 +63,9 @@ D := 0
 
 ifeq (\$D,1)
         CFLAGS += -O0
+        RUST_FLAGS += --debug
+else
+        RUST_FLAGS += --release
 endif
 
 # Optionally disable -Werror with
@@ -76,6 +80,18 @@ END
 
 fi
 
+# potentially set otel lib location
+if [ $NXT_OTEL = YES ]; then
+cat << END >> $NXT_MAKEFILE
+
+ifeq (\$D,1)
+	NXT_OTEL_LIB_LOC = $NXT_OTEL_LIB_DIR/target/debug/libotel.a
+else
+	NXT_OTEL_LIB_LOC = $NXT_OTEL_LIB_DIR/target/release/libotel.a
+endif
+
+END
+fi
 
 # The include paths list.
 
@@ -138,14 +154,14 @@ cat << END >> $NXT_MAKEFILE
 
 libnxt:	$NXT_BUILD_DIR/lib/$NXT_LIB_SHARED $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC
 
-$NXT_BUILD_DIR/lib/$NXT_LIB_SHARED: \$(NXT_LIB_OBJS)
+$NXT_BUILD_DIR/lib/$NXT_LIB_SHARED: \$(NXT_LIB_OBJS) \$(NXT_OTEL_LIB_LOC)
 	\$(PP_LD) \$@
 	\$(v)\$(NXT_SHARED_LOCAL_LINK) -o \$@ \$(NXT_LIB_OBJS) \\
-		$NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS
+		$NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \$(NXT_OTEL_LIB_LOC)
 
-$NXT_BUILD_DIR/lib/$NXT_LIB_STATIC: \$(NXT_LIB_OBJS)
+$NXT_BUILD_DIR/lib/$NXT_LIB_STATIC: \$(NXT_LIB_OBJS) \$(NXT_OTEL_LIB_LOC)
 	\$(PP_AR) \$@
-	\$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_OBJS)
+	\$(v)$NXT_STATIC_LINK \$@ \$(NXT_LIB_OBJS) \$(NXT_OTEL_LIB_LOC)
 
 $NXT_BUILD_DIR/lib/$NXT_LIB_UNIT_STATIC: \$(NXT_LIB_UNIT_OBJS) \\
 		$NXT_BUILD_DIR/share/pkgconfig/unit.pc \\
@@ -359,11 +375,11 @@ $echo >> $NXT_MAKEFILE
 cat << END >> $NXT_MAKEFILE
 
 $NXT_BUILD_DIR/sbin/$NXT_DAEMON:	$NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\
-				\$(NXT_OBJS)
+				\$(NXT_OBJS) \$(NXT_OTEL_LIB_LOC)
 	\$(PP_LD) \$@
 	\$(v)\$(NXT_EXEC_LINK) -o \$@ \$(CFLAGS) \\
 		\$(NXT_OBJS) $NXT_BUILD_DIR/lib/$NXT_LIB_STATIC \\
-		$NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS
+		$NXT_LIBM $NXT_LIBS $NXT_LIB_AUX_LIBS \$(NXT_OTEL_LIB_LOC)
 
 END
 
@@ -535,10 +551,6 @@ cat << END > Makefile
 
 include $NXT_MAKEFILE
 
-.PHONY: clean
-clean:
-		rm -rf $NXT_BUILD_DIR *.dSYM Makefile
-
 .PHONY: help
 help:
 	@echo "Variables to control make/build behaviour:"
@@ -551,4 +563,24 @@ help:
 	@echo
 	@echo "  Variables can be combined."
 
+.PHONY: clean
+clean:
+		rm -rf $NXT_BUILD_DIR *.dSYM Makefile
+END
+
+if [ $NXT_OTEL = YES ]; then
+    cat << END >> Makefile
+		cd "$NXT_OTEL_LIB_DIR" && cargo clean
 END
+
+NXT_OTEL_DEPS=" \
+    build/src/nxt_otel.o \
+    src/otel/src/lib.rs \
+"
+
+    cat << END >> $NXT_MAKEFILE
+
+\$(NXT_OTEL_LIB_LOC): $NXT_OTEL_DEPS
+	cargo build \$(RUST_FLAGS) --manifest-path $NXT_OTEL_LIB_DIR/Cargo.toml
+END
+fi
diff --git a/auto/options b/auto/options
index 5be1ebe1..7aa7a73a 100644
--- a/auto/options
+++ b/auto/options
@@ -27,6 +27,7 @@ NXT_CYASSL=NO
 NXT_POLARSSL=NO
 
 NXT_NJS=NO
+NXT_OTEL=NO
 
 NXT_TEST_BUILD_EPOLL=NO
 NXT_TEST_BUILD_EVENTPORT=NO
@@ -112,6 +113,7 @@ do
         --polarssl)                      NXT_POLARSSL=YES                    ;;
 
         --njs)                           NXT_NJS=YES                         ;;
+        --otel)                          NXT_OTEL=YES                        ;;
 
         --test-build-epoll)              NXT_TEST_BUILD_EPOLL=YES            ;;
         --test-build-eventport)          NXT_TEST_BUILD_EVENTPORT=YES        ;;
diff --git a/auto/otel b/auto/otel
new file mode 100644
index 00000000..654c45ec
--- /dev/null
+++ b/auto/otel
@@ -0,0 +1,54 @@
+
+# Copyright (C) NGINX, Inc.
+
+if [ $NXT_OTEL = YES ]; then
+
+    $echo "checking for OTEL requirements:"
+
+    $echo -n "  - checking for rust compiler ... "
+    if [ -z $(which rustc 2>/dev/null) ]; then
+        $echo "not found"
+        exit 1;
+    fi
+    $echo "found"
+
+    $echo -n "  - checking for cargo ... "
+    if [ -z $(which cargo 2>/dev/null) ]; then
+        $echo "not found."
+        exit 1;
+    fi
+    $echo "found"
+
+    $echo -n "  - "
+
+    nxt_feature="OpenSSL library"
+    nxt_feature_run=yes
+    nxt_feature_incs=
+    nxt_feature_libs="-lssl -lcrypto"
+    nxt_feature_test="#include <openssl/ssl.h>
+
+                      int main(void) {
+                          SSL_library_init();
+                          return 0;
+                      }"
+    . auto/feature
+
+    if [ ! $nxt_found = yes ]; then
+        $echo
+        $echo $0: error: OpenTelemetry support requires OpenSSL.
+        $echo
+        exit 1;
+    fi
+
+    NXT_OTEL_LIBS="-lssl -lcrypto"
+    if [ $(which pkg-config) ]; then
+        NXT_OTEL_LIBS="$(pkg-config openssl --cflags --libs)"
+    fi
+    cat << END >> $NXT_AUTO_CONFIG_H
+
+#ifndef NXT_HAVE_OTEL
+#define NXT_HAVE_OTEL 1
+#endif
+
+END
+fi
diff --git a/auto/sources b/auto/sources
index dfabf7cf..02740397 100644
--- a/auto/sources
+++ b/auto/sources
@@ -127,6 +127,10 @@ if [ "$NXT_NJS" != "NO" ]; then
     NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_js.c src/nxt_http_js.c src/nxt_script.c"
 fi
 
+if [ "$NXT_OTEL" != "NO" ]; then
+    NXT_LIB_SRCS="$NXT_LIB_SRCS src/nxt_otel.c"
+fi
+
 NXT_LIB_EPOLL_SRCS="src/nxt_epoll_engine.c"
 NXT_LIB_KQUEUE_SRCS="src/nxt_kqueue_engine.c"
 NXT_LIB_EVENTPORT_SRCS="src/nxt_eventport_engine.c"
diff --git a/auto/summary b/auto/summary
index b6caee6c..eba88be4 100644
--- a/auto/summary
+++ b/auto/summary
@@ -30,6 +30,7 @@ Unit configuration summary:
   TLS support: ............... $NXT_OPENSSL
   Regex support: ............. $NXT_REGEX
   njs support: ............... $NXT_NJS
+  otel support: .............. $NXT_OTEL
 
   process isolation: ......... $NXT_ISOLATION
   cgroupv2: .................. $NXT_HAVE_CGROUP
diff --git a/configure b/configure
index 6929d41d..05a992ee 100755
--- a/configure
+++ b/configure
@@ -179,6 +179,12 @@ if [ $NXT_NJS != NO ]; then
     . auto/njs
 fi
 
+NXT_OTEL_LIB_DIR=src/otel
+if [ $NXT_OTEL != NO ]; then
+    . auto/otel
+    NXT_LIB_AUX_LIBS="$NXT_LIB_AUX_LIBS $NXT_OTEL_LIBS"
+fi
+
 . auto/make
 . auto/fuzzing
 . auto/summary
diff --git a/src/nxt_http.h b/src/nxt_http.h
index 5369c8e1..19bbdda3 100644
--- a/src/nxt_http.h
+++ b/src/nxt_http.h
@@ -8,6 +8,7 @@
 #define _NXT_HTTP_H_INCLUDED_
 
 #include <nxt_regex.h>
+#include <nxt_otel.h>
 
 
 typedef enum {
@@ -190,6 +191,10 @@ struct nxt_http_request_s {
 
     nxt_http_response_t             resp;
 
+#if (NXT_HAVE_OTEL)
+    nxt_otel_state_t                *otel;
+#endif
+
     nxt_http_status_t               status:16;
 
     uint8_t                         log_route;    /* 1 bit */
diff --git a/src/nxt_http_error.c b/src/nxt_http_error.c
index 370b12db..4cafb872 100644
--- a/src/nxt_http_error.c
+++ b/src/nxt_http_error.c
@@ -6,6 +6,7 @@
 
 #include <nxt_router.h>
 #include <nxt_http.h>
+#include <nxt_otel.h>
 
 
 static void nxt_http_request_send_error_body(nxt_task_t *task, void *r,
@@ -55,6 +56,10 @@ nxt_http_request_error(nxt_task_t *task, nxt_http_request_t *r,
     r->resp.content_length = NULL;
     r->resp.content_length_n = NXT_HTTP_ERROR_LEN;
 
+#if (NXT_HAVE_OTEL)
+    nxt_otel_request_error_path(task, r);
+#endif
+
     r->state = &nxt_http_request_send_error_body_state;
 
     nxt_http_request_header_send(task, r,
diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c
index a7e9ff69..417b5a39 100644
--- a/src/nxt_http_request.c
+++ b/src/nxt_http_request.c
@@ -6,6 +6,7 @@
 
 #include <nxt_router.h>
 #include <nxt_http.h>
+#include <nxt_otel.h>
 
 
 static nxt_int_t nxt_http_validate_host(nxt_str_t *host, nxt_mp_t *mp);
@@ -284,6 +285,16 @@ nxt_http_request_create(nxt_task_t *task)
 
     r->tstr_cache.var.pool = mp;
 
+#if (NXT_HAVE_OTEL)
+    if (nxt_otel_rs_is_init()) {
+        r->otel = nxt_mp_zget(r->mem_pool, sizeof(nxt_otel_state_t));
+        if (nxt_slow_path(r->otel == NULL)) {
+            goto fail;
+        }
+        r->otel->status = NXT_OTEL_INIT_STATE;
+    }
+#endif
+
     return r;
 
 fail:
diff --git a/src/nxt_otel.c b/src/nxt_otel.c
new file mode 100644
index 00000000..eea12e24
--- /dev/null
+++ b/src/nxt_otel.c
@@ -0,0 +1,416 @@
+
+/*
+ * Copyright (C) F5, Inc.
+ */
+
+#include <math.h>
+
+#include <nxt_router.h>
+#include <nxt_http.h>
+#include <nxt_otel.h>
+#include <nxt_mp.h>
+#include <nxt_work_queue.h>
+#include <nxt_main.h>
+#include <nxt_conf.h>
+#include <nxt_types.h>
+#include <nxt_string.h>
+#include <nxt_clang.h>
+
+
+#define NXT_OTEL_TRACEPARENT_LEN    55
+#define NXT_OTEL_BODY_SIZE_TAG      "body size"
+#define NXT_OTEL_METHOD_TAG         "method"
+#define NXT_OTEL_PATH_TAG           "path"
+#define NXT_OTEL_STATUS_CODE_TAG    "status"
+
+
+static void
+nxt_otel_state_transition(nxt_otel_state_t *state, nxt_otel_status_t status)
+{
+    if (status == NXT_OTEL_ERROR_STATE
+        || state->status != NXT_OTEL_ERROR_STATE)
+    {
+        state->status = status;
+    }
+}
+
+
+static void
+nxt_otel_propagate_header(nxt_task_t *task, nxt_http_request_t *r)
+{
+    u_char            *traceval;
+    nxt_str_t         traceparent_name, traceparent;
+    nxt_http_field_t  *f;
+
+    traceval = nxt_mp_zalloc(r->mem_pool, NXT_OTEL_TRACEPARENT_LEN + 1);
+    if (nxt_slow_path(traceval == NULL)) {
+        /*
+         * let it go blank here.
+         * span still gets populated and sent
+         * but data is not propagated to peer or app.
+         */
+        nxt_log(task, NXT_LOG_ERR,
+                "couldn't allocate traceparent header. "
+                "span will not propagate");
+        return;
+    }
+
+    if (r->otel->trace_id != NULL) {
+        // copy in the pre-existing traceparent for the response
+        sprintf((char *) traceval, "%s-%s-%s-%s",
+                (char *) r->otel->version,
+                (char *) r->otel->trace_id,
+                (char *) r->otel->parent_id,
+                (char *) r->otel->trace_flags);
+
+    /*
+     * if we didn't inherit a trace id then we need to add the
+     * traceparent header to the request
+     */
+    } else if (r->otel->trace_id == NULL) {
+
+        nxt_otel_rs_copy_traceparent(traceval, r->otel->trace);
+
+        f = nxt_list_add(r->fields);
+        if (nxt_slow_path(f == NULL)) {
+            return;
+        }
+
+        nxt_http_field_name_set(f, "traceparent");
+        f->value = traceval;
+        f->value_length = nxt_strlen(traceval);
+
+        traceparent_name = (nxt_str_t) {
+            .start  = f->name,
+            .length = f->name_length,
+        };
+
+        traceparent = (nxt_str_t) {
+            .start  = f->value,
+            .length = f->value_length,
+        };
+
+        nxt_otel_rs_add_event_to_trace(r->otel->trace,
+                                       &traceparent_name, &traceparent);
+
+    /*
+     * potentially nxt_http_request_error called before headers
+     * finished parsing
+     */
+    } else {
+        nxt_log(task, NXT_LOG_DEBUG,
+                "not propagating tracing headers for missing trace");
+        return;
+    }
+
+    f = nxt_list_add(r->resp.fields);
+    if (nxt_slow_path(f == NULL)) {
+        nxt_log(task, NXT_LOG_ERR,
+                "couldn't allocate traceparent header in response");
+        return;
+    }
+
+    nxt_http_field_name_set(f, "traceparent");
+    f->value = traceval;
+    f->value_length = nxt_strlen(traceval);
+}
+
+
+static void
+nxt_otel_span_add_headers(nxt_task_t *task, nxt_http_request_t *r)
+{
+    nxt_str_t         method_name, path_name;
+    nxt_http_field_t  *cur;
+
+    nxt_log(task, NXT_LOG_DEBUG, "adding headers to trace");
+
+    if (r->otel == NULL || r->otel->trace == NULL) {
+        nxt_log(task, NXT_LOG_ERR, "no trace to add events to!");
+        nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE);
+        return;
+    }
+
+    nxt_list_each(cur, r->fields) {
+        nxt_str_t  name, val;
+
+        name = (nxt_str_t) {
+            .start  = cur->name,
+            .length = cur->name_length,
+        };
+
+        val = (nxt_str_t) {
+            .start  = cur->value,
+            .length = cur->value_length,
+        };
+
+        nxt_otel_rs_add_event_to_trace(r->otel->trace, &name, &val);
+    } nxt_list_loop;
+
+    nxt_str_set(&method_name, NXT_OTEL_METHOD_TAG);
+    nxt_otel_rs_add_event_to_trace(r->otel->trace, &method_name, r->method);
+    nxt_str_set(&path_name, NXT_OTEL_PATH_TAG);
+    nxt_otel_rs_add_event_to_trace(r->otel->trace, &path_name, r->path);
+    nxt_otel_propagate_header(task, r);
+
+    nxt_otel_state_transition(r->otel, NXT_OTEL_BODY_STATE);
+}
+
+
+static void
+nxt_otel_span_add_body(nxt_http_request_t *r)
+{
+    size_t     body_size = 0;
+    size_t     buf_size;
+    u_char     *body_buf, *body_size_buf;
+    nxt_int_t  cur;
+    nxt_str_t  body_key, body_val;
+
+    if (r->body != NULL) {
+        body_size = nxt_buf_used_size(r->body);
+    }
+
+    buf_size = 1; // first digit
+    if (body_size != 0) {
+        buf_size += log10(body_size); // subsequent digits
+    }
+    buf_size += 1; // \0
+    buf_size += nxt_strlen(NXT_OTEL_BODY_SIZE_TAG);
+    buf_size += 1; // \0
+
+    body_buf = nxt_mp_alloc(r->mem_pool, buf_size);
+    if (nxt_slow_path(body_buf == NULL)) {
+        return;
+    }
+
+    cur = sprintf((char *) body_buf, "%lu", body_size);
+    if (cur < 0) {
+        return;
+    }
+
+    cur += 1;
+    body_size_buf = body_buf + cur;
+    nxt_cpystr(body_buf + cur, (const u_char *) NXT_OTEL_BODY_SIZE_TAG);
+
+    body_key = (nxt_str_t) {
+        .start  = body_size_buf,
+        .length = nxt_strlen(body_size_buf),
+    };
+
+    body_val = (nxt_str_t) {
+        .start  = body_buf,
+        .length = nxt_strlen(body_buf),
+    };
+
+    nxt_otel_rs_add_event_to_trace(r->otel->trace, &body_key, &body_val);
+    nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE);
+}
+
+
+static void
+nxt_otel_span_add_status(nxt_task_t *task, nxt_http_request_t *r)
+{
+    u_char     status_buf[7];
+    nxt_str_t  status_key, status_val;
+
+    // dont bother logging an unset status
+    if (r->status == 0) {
+        return;
+    }
+
+    sprintf((char *) status_buf, "%d", r->status);
+
+    // set up event
+    nxt_str_set(&status_key, NXT_OTEL_STATUS_CODE_TAG);
+
+    status_val = (nxt_str_t) {
+        .start  = status_buf,
+        .length = 3,
+    };
+
+    nxt_otel_rs_add_event_to_trace(r->otel->trace, &status_key, &status_val);
+}
+
+
+static void
+nxt_otel_span_collect(nxt_task_t *task, nxt_http_request_t *r)
+{
+    if (r->otel->trace == NULL) {
+        nxt_log(task, NXT_LOG_ERR, "otel error: no trace to send!");
+        nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE);
+        return;
+    }
+
+    nxt_otel_span_add_status(task, r);
+    nxt_otel_state_transition(r->otel, NXT_OTEL_UNINIT_STATE);
+    nxt_otel_rs_send_trace(r->otel->trace);
+
+    r->otel->trace = NULL;
+}
+
+
+static void
+nxt_otel_error(nxt_task_t *task, nxt_http_request_t *r)
+{
+    // purposefully not using state transition helper
+    r->otel->status = NXT_OTEL_UNINIT_STATE;
+    nxt_log(task, NXT_LOG_ERR, "otel error condition");
+
+    /*
+     * assumable at time of writing that there is no
+     * r->otel->trace to leak. This state is only set
+     * in cases where trace fails to generate or is missing
+     */
+}
+
+
+static void
+nxt_otel_trace_and_span_init(nxt_task_t *task, nxt_http_request_t *r)
+{
+    r->otel->trace =
+        nxt_otel_rs_get_or_create_trace(r->otel->trace_id);
+    if (r->otel->trace == NULL) {
+        nxt_log(task, NXT_LOG_ERR, "error generating otel span");
+        nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE);
+        return;
+    }
+
+    nxt_otel_state_transition(r->otel, NXT_OTEL_HEADER_STATE);
+}
+
+
+void
+nxt_otel_test_and_call_state(nxt_task_t *task, nxt_http_request_t *r)
+{
+    if (r == NULL || r->otel == NULL) {
+        return;
+    }
+
+    switch (r->otel->status) {
+    case NXT_OTEL_UNINIT_STATE:
+        return;
+    case NXT_OTEL_INIT_STATE:
+        nxt_otel_trace_and_span_init(task, r);
+        break;
+    case NXT_OTEL_HEADER_STATE:
+        nxt_otel_span_add_headers(task, r);
+        break;
+    case NXT_OTEL_BODY_STATE:
+        nxt_otel_span_add_body(r);
+        break;
+    case NXT_OTEL_COLLECT_STATE:
+        nxt_otel_span_collect(task, r);
+        break;
+    case NXT_OTEL_ERROR_STATE:
+        nxt_otel_error(task, r);
+        break;
+    }
+}
+
+
+// called in nxt_http_request_error
+void
+nxt_otel_request_error_path(nxt_task_t *task, nxt_http_request_t *r)
+{
+    if (r->otel->trace == NULL) {
+        return;
+    }
+
+    // response headers have been cleared
+    nxt_otel_propagate_header(task, r);
+
+    // collect span immediately
+    if (r->otel) {
+        nxt_otel_state_transition(r->otel, NXT_OTEL_COLLECT_STATE);
+    }
+    nxt_otel_test_and_call_state(task, r);
+}
+
+
+nxt_int_t
+nxt_otel_parse_traceparent(void *ctx, nxt_http_field_t *field, uintptr_t data)
+{
+    char                *copy;
+    nxt_http_request_t  *r;
+
+    /*
+     * For information on parsing the traceparent header:
+     * https://www.w3.org/TR/trace-context/#traceparent-header
+     * A summary of the traceparent header value format follows:
+     * Traceparent: "$a-$b-$c-$d"
+     *   a. version (2 hex digits) (ff is forbidden)
+     *   b. trace_id (32 hex digits) (all zeroes forbidden)
+     *   c. parent_id (16 hex digits) (all zeroes forbidden)
+     *   d. flags (2 hex digits)
+     */
+
+    r = ctx;
+
+    if (field->value_length != NXT_OTEL_TRACEPARENT_LEN) {
+        goto error_state;
+    }
+
+    /*
+     * strsep is destructive so we make a copy of the field
+     */
+    copy = nxt_mp_zalloc(r->mem_pool, field->value_length + 1);
+    if (nxt_slow_path(copy == NULL)) {
+        goto error_state;
+    }
+    memcpy(copy, field->value, field->value_length);
+
+    r->otel->version = (u_char *) strsep(&copy, "-");
+    r->otel->trace_id = (u_char *) strsep(&copy, "-");
+    r->otel->parent_id = (u_char *) strsep(&copy, "-");
+    r->otel->trace_flags = (u_char *) strsep(&copy, "-");
+
+    if (r->otel->version        == NULL
+        || r->otel->trace_id    == NULL
+        || r->otel->parent_id   == NULL
+        || r->otel->trace_flags == NULL)
+    {
+        goto error_state;
+    }
+
+    return NXT_OK;
+
+error_state:
+    nxt_otel_state_transition(r->otel, NXT_OTEL_ERROR_STATE);
+
+    return NXT_ERROR;
+}
+
+
+nxt_int_t
+nxt_otel_parse_tracestate(void *ctx, nxt_http_field_t *field, uintptr_t data)
+{
+    nxt_str_t           s;
+    nxt_http_field_t    *f;
+    nxt_http_request_t  *r;
+
+    s.length = field->value_length;
+    s.start = field->value;
+
+    r = ctx;
+    r->otel->trace_state = s;
+
+    /*
+     * maybe someday this should get sent down into the otel lib
+     * when we can figure out what to do with it at least
+     */
+
+    f = nxt_list_add(r->resp.fields);
+    if (nxt_fast_path(f != NULL)) {
+        *f = *field;
+    }
+
+    return NXT_OK;
+}
+
+
+void
+nxt_otel_log_callback(nxt_uint_t log_level, const char *arg)
+{
+    nxt_thread_t  *thr = nxt_thread();
+
+    nxt_log(thr->task, log_level, "otel: %s", arg);
+}
diff --git a/src/nxt_otel.h b/src/nxt_otel.h
new file mode 100644
index 00000000..f5a2de8b
--- /dev/null
+++ b/src/nxt_otel.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) F5, Inc.
+ */
+
+#ifndef _NXT_OTEL_H_INCLUDED_
+#define _NXT_OTEL_H_INCLUDED_
+
+#include <nxt_router.h>
+#include <nxt_string.h>
+
+
+#if (NXT_HAVE_OTEL)
+extern void nxt_otel_rs_send_trace(void *trace);
+extern void * nxt_otel_rs_get_or_create_trace(u_char *trace_id);
+extern void nxt_otel_rs_init(
+    void (*log_callback)(nxt_uint_t log_level, const char *log_string),
+    const nxt_str_t *endpoint, const nxt_str_t *protocol,
+    double sample_fraction, double batch_size);
+extern void nxt_otel_rs_copy_traceparent(u_char *buffer, void *span);
+extern void nxt_otel_rs_add_event_to_trace(void *trace, nxt_str_t *key,
+    nxt_str_t *val);
+extern uint8_t nxt_otel_rs_is_init(void);
+extern void nxt_otel_rs_uninit(void);
+#endif
+
+
+typedef enum nxt_otel_status_e   nxt_otel_status_t;
+typedef struct nxt_otel_state_s  nxt_otel_state_t;
+
+
+/*
+ * nxt_otel_status_t
+ * more efficient than a single handler state struct
+ */
+enum nxt_otel_status_e {
+    NXT_OTEL_UNINIT_STATE = 0,
+    NXT_OTEL_INIT_STATE,
+    NXT_OTEL_HEADER_STATE,
+    NXT_OTEL_BODY_STATE,
+    NXT_OTEL_COLLECT_STATE,
+    NXT_OTEL_ERROR_STATE,
+};
+
+
+/*
+ * nxt_otel_state_t
+ * cache of trace data needed per request and
+ * includes indicator as to current flow state
+ */
+struct nxt_otel_state_s {
+    u_char             *trace_id;
+    u_char             *version;
+    u_char             *parent_id;
+    u_char             *trace_flags;
+    void               *trace;
+    nxt_otel_status_t  status;
+    nxt_str_t          trace_state;
+};
+
+
+nxt_int_t nxt_otel_parse_traceparent(void *ctx, nxt_http_field_t *field,
+    uintptr_t data);
+nxt_int_t nxt_otel_parse_tracestate(void *ctx, nxt_http_field_t *field,
+    uintptr_t data);
+void nxt_otel_log_callback(nxt_uint_t log_level, const char *arg);
+
+
+void nxt_otel_test_and_call_state(nxt_task_t *task, nxt_http_request_t *r);
+void nxt_otel_request_error_path(nxt_task_t *task, nxt_http_request_t *r);
+
+
+#endif /* _NXT_OTEL_H_INCLUDED_ */