diff --git a/README b/README index 2f68e14..262822a 100644 --- a/README +++ b/README @@ -1,3 +1,38 @@ +This is an Nginx fork that adds dtrace USDT probes. -Documentation is available at http://nginx.org +Installation: + + ./configure --with-dtrace-probes \ + --with-dtrace=/usr/sbin/dtrace \ + ... + make + make install + +Usage on Linux (with systemtap): + + # make the stap-nginx script visiable in your PATH + export PATH=/usr/local/nginx/sbin:$PATH + + # list all the static probes available in your nginx + stap-nginx -L 'process("nginx").mark("*")' + + # run the test.stp file + stap-nginx test.stp + +Sample test.stp file: + + probe begin + { + print("Tracing. Hit CTRL-C to stop.\n") + } + + probe process("nginx").mark("http-subrequest-start") + { + printf("uri: %s?%s\n", ngx_http_req_uri($arg1), + ngx_http_req_args($arg1)) + } + +For now, only tested on Solaris 11 Express and Fedora Linux 17. + +The original Nginx documentation is available at http://nginx.org diff --git a/auto/install b/auto/install index 254f9bc..0252fef 100644 --- a/auto/install +++ b/auto/install @@ -16,6 +16,20 @@ END fi +case ".$NGX_STAP_NGX_PATH" in + ./*) + ;; + + .) + NGX_STAP_NGX_PATH=$NGX_PREFIX/sbin/stap-nginx + ;; + + *) + NGX_STAP_NGX_PATH=$NGX_PREFIX/$NGX_STAP_NGX_PATH + ;; +esac + + case ".$NGX_SBIN_PATH" in ./*) ;; @@ -53,6 +67,16 @@ case ".$NGX_PID_PATH" in esac +case ".$NGX_TAPSET_PREFIX" in + ./* | .) + ;; + + *) + NGX_TAPSET_PREFIX=$NGX_PREFIX/$NGX_TAPSET_PREFIX + ;; +esac + + case ".$NGX_ERROR_LOG_PATH" in ./* | .) ;; @@ -151,6 +175,36 @@ install: $NGX_OBJS${ngx_dirsep}nginx${ngx_binext} \ || cp -R $NGX_HTML '\$(DESTDIR)$NGX_PREFIX' END +if [ $NGX_DTRACE = YES -a $DTRACE_FROM_SYSTEMTAP = YES ]; then + + ngx_tapset_srcs="$NGX_TAPSET_SRCS" + + cat << END >> $NGX_MAKEFILE + test -d '\$(DESTDIR)$NGX_TAPSET_PREFIX' || \ + mkdir -p '\$(DESTDIR)$NGX_TAPSET_PREFIX' +END + + for ngx_tapset_src in $ngx_tapset_srcs + do + ngx_tapset_file=`basename $ngx_tapset_src` + + cat << END >> $NGX_MAKEFILE + + sed -e "s|NGX_SBIN_PATH|$NGX_SBIN_PATH|g" $ngx_long_cont \ + $ngx_tapset_src > '\$(DESTDIR)$NGX_TAPSET_PREFIX/$ngx_tapset_file' +END + + done + + cat << END >> $NGX_MAKEFILE + + test -d '\$(DESTDIR)`dirname "$NGX_STAP_NGX_PATH"`' || \ + mkdir -p '\$(DESTDIR)`dirname "$NGX_STAP_NGX_PATH"`' + cp $NGX_OBJS/stap-nginx '\$(DESTDIR)$NGX_STAP_NGX_PATH' + chmod 0755 '\$(DESTDIR)$NGX_STAP_NGX_PATH' +END + +fi if test -n "$NGX_ERROR_LOG_PATH"; then cat << END >> $NGX_MAKEFILE @@ -162,6 +216,18 @@ END fi +if [ $NGX_DTRACE = YES ]; then + cat << END >> $NGX_MAKEFILE + +$NGX_OBJS${ngx_dirsep}stap-nginx: src/dtrace/stap-nginx + sed -e "s|NGX_TAPSET_PREFIX|$NGX_TAPSET_PREFIX|g" $ngx_long_cont \ + -e "s|NGX_SBIN_DIR|`dirname $NGX_SBIN_PATH`|g" $ngx_long_cont \ + -e "s|NGX_SBIN_PATH|$NGX_SBIN_PATH|g" $ngx_long_cont \ + src/dtrace/stap-nginx > $NGX_OBJS${ngx_dirsep}stap-nginx +END +fi + + # create Makefile cat << END >> Makefile diff --git a/auto/make b/auto/make index 05b7454..aa813b1 100644 --- a/auto/make +++ b/auto/make @@ -26,6 +26,9 @@ LINK = $LINK END +if [ $NGX_DTRACE = YES ]; then + echo DTRACE = $DTRACE >> $NGX_MAKEFILE +fi if test -n "$NGX_PERL_CFLAGS"; then echo NGX_PERL_CFLAGS = $NGX_PERL_CFLAGS >> $NGX_MAKEFILE @@ -177,6 +180,44 @@ ngx_objs=`echo $ngx_all_objs $ngx_modules_obj \ | sed -e "s/ *\([^ ][^ ]*\)/$ngx_long_regex_cont\1/g" \ -e "s/\//$ngx_regex_dirsep/g"` +if [ $NGX_DTRACE = YES ]; then + + ngx_dtrace_obj=$NGX_OBJS${ngx_dirsep}ngx_dtrace_provider.$ngx_objext + + ngx_dtrace_h=$NGX_OBJS${ngx_dirsep}ngx_dtrace_provider.h + + ngx_dtrace_d=$NGX_OBJS${ngx_dirsep}dtrace_providers.d + + ngx_dtrace_providers=`echo $NGX_DTRACE_PROVIDERS \ + | sed -e "s/ *\([^ ][^ ]*\)/$ngx_long_regex_cont\1/g" \ + -e "s/\//$ngx_regex_dirsep/g"` + + cat << END >> $NGX_MAKEFILE + +all: $NGX_OBJS${ngx_dirsep}nginx${ngx_binext} + +$ngx_dtrace_d: $ngx_dtrace_providers + cat $ngx_dtrace_providers > $ngx_dtrace_d + +$ngx_dtrace_h: $ngx_dtrace_d + \$(DTRACE) -xnolibs -h -o $ngx_dtrace_h -s $ngx_dtrace_d +END + + if [ $DTRACE_PROBE_OBJ = YES ]; then + cat << END >> $NGX_MAKEFILE +$ngx_dtrace_obj: $ngx_dtrace_d $ngx_deps$ngx_spacer + \$(DTRACE) -xnolibs -G -o $ngx_dtrace_obj -s $ngx_dtrace_d $ngx_objs +END + + ngx_deps="$ngx_deps$ngx_long_cont$ngx_dtrace_obj" + ngx_objs="$ngx_objs$ngx_long_cont$ngx_dtrace_obj" + + if [ "$DTRACE_FROM_SYSTEMTAP" = YES ]; then + ngx_deps="$ngx_deps$ngx_long_cont$NGX_OBJS${ngx_dirsep}stap-nginx" + fi + fi +fi + if test -n "$NGX_LD_OPT$CORE_LIBS"; then ngx_libs=`echo $NGX_LD_OPT $CORE_LIBS \ | sed -e "s/\//$ngx_regex_dirsep/g" -e "s/^/$ngx_long_regex_cont/"` diff --git a/auto/options b/auto/options index 6c3a4db..8d18c01 100644 --- a/auto/options +++ b/auto/options @@ -12,6 +12,8 @@ NGX_CONF_PATH= NGX_ERROR_LOG_PATH= NGX_PID_PATH= NGX_LOCK_PATH= +NGX_TAPSET_PREFIX= +NGX_STAP_NGX_PATH= NGX_USER= NGX_GROUP= @@ -20,6 +22,12 @@ CPP= NGX_OBJS=objs NGX_DEBUG=NO +NGX_DTRACE=NO +DTRACE=dtrace + +DTRACE_PROBE_OBJ=YES +DTRACE_FROM_SYSTEMTAP=NO + NGX_CC_OPT= NGX_LD_OPT= CPU=NO @@ -170,6 +178,8 @@ do --error-log-path=*) NGX_ERROR_LOG_PATH="$value";; --pid-path=*) NGX_PID_PATH="$value" ;; --lock-path=*) NGX_LOCK_PATH="$value" ;; + --tapset-prefix=*) NGX_TAPSET_PREFIX="$value" ;; + --stap-nginx-path=*) NGX_STAP_NGX_PATH="$value" ;; --user=*) NGX_USER="$value" ;; --group=*) NGX_GROUP="$value" ;; @@ -275,7 +285,8 @@ use the \"--without-http_limit_conn_module\" option instead" --with-ld-opt=*) NGX_LD_OPT="$value" ;; --with-cpu-opt=*) CPU="$value" ;; --with-debug) NGX_DEBUG=YES ;; - + --with-dtrace=*) DTRACE="$value" ;; + --with-dtrace-probes) NGX_DTRACE=YES ;; --without-pcre) USE_PCRE=DISABLED ;; --with-pcre) USE_PCRE=YES ;; --with-pcre=*) PCRE="$value" ;; @@ -329,6 +340,8 @@ cat << END --error-log-path=PATH set error log pathname --pid-path=PATH set nginx.pid pathname --lock-path=PATH set nginx.lock pathname + --tapset-prefix=PATH set systemtap tapset directory prefix + --stap-nginx-path=PATH set stap-nginx pathname --user=USER set non-privileged user for worker processes @@ -455,6 +468,8 @@ cat << END --with-openssl-opt=OPTIONS set additional build options for OpenSSL --with-debug enable debug logging + --with-dtrace-probes enable dtrace USDT probes + --with-dtrace=PATH set dtrace utility pathname END @@ -484,6 +499,7 @@ NGX_CONF_PATH=${NGX_CONF_PATH:-conf/nginx.conf} NGX_CONF_PREFIX=`dirname $NGX_CONF_PATH` NGX_PID_PATH=${NGX_PID_PATH:-logs/nginx.pid} NGX_LOCK_PATH=${NGX_LOCK_PATH:-logs/nginx.lock} +NGX_TAPSET_PREFIX=${NGX_TAPSET_PREFIX:-tapset} if [ ".$NGX_ERROR_LOG_PATH" = ".stderr" ]; then NGX_ERROR_LOG_PATH= diff --git a/auto/os/darwin b/auto/os/darwin index 590e036..af33cf3 100644 --- a/auto/os/darwin +++ b/auto/os/darwin @@ -114,3 +114,6 @@ ngx_feature_libs= ngx_feature_test="int32_t lock, n; n = OSAtomicCompareAndSwap32Barrier(0, 1, lock)" . auto/feature + +DTRACE_PROBE_OBJ=NO + diff --git a/auto/os/freebsd b/auto/os/freebsd index 6aa823f..ee15166 100644 --- a/auto/os/freebsd +++ b/auto/os/freebsd @@ -142,3 +142,8 @@ if [ $version -ge 701000 ]; then echo " + cpuset_setaffinity() found" have=NGX_HAVE_CPUSET_SETAFFINITY . auto/have fi + +if [ $NGX_DTRACE = YES ]; then + NGX_LD_OPT="$NGX_LD_OPT -lelf" +fi + diff --git a/auto/os/linux b/auto/os/linux index c506d3d..cbbfbf1 100644 --- a/auto/os/linux +++ b/auto/os/linux @@ -151,3 +151,5 @@ ngx_include="sys/vfs.h"; . auto/include CC_AUX_FLAGS="$cc_aux_flags -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64" + +DTRACE_FROM_SYSTEMTAP=YES diff --git a/auto/sources b/auto/sources index 522d722..5e48797 100644 --- a/auto/sources +++ b/auto/sources @@ -36,7 +36,13 @@ CORE_DEPS="src/core/nginx.h \ src/core/ngx_conf_file.h \ src/core/ngx_resolver.h \ src/core/ngx_open_file_cache.h \ - src/core/ngx_crypt.h" + src/core/ngx_crypt.h \ + src/core/ngx_core_probe.h" + + +if [ $NGX_DTRACE = YES ]; then + CORE_DEPS="$CORE_DEPS objs/ngx_dtrace_provider.h" +fi CORE_SRCS="src/core/nginx.c \ @@ -82,14 +88,15 @@ OPENSSL_SRCS=src/event/ngx_event_openssl.c EVENT_MODULES="ngx_events_module ngx_event_core_module" -EVENT_INCS="src/event src/event/modules" +EVENT_INCS="src/event src/event/modules src/http src/http/modules" EVENT_DEPS="src/event/ngx_event.h \ src/event/ngx_event_timer.h \ src/event/ngx_event_posted.h \ src/event/ngx_event_busy_lock.h \ src/event/ngx_event_connect.h \ - src/event/ngx_event_pipe.h" + src/event/ngx_event_pipe.h \ + src/event/ngx_event_probe.h" EVENT_SRCS="src/event/ngx_event.c \ src/event/ngx_event_timer.c \ @@ -291,7 +298,8 @@ HTTP_DEPS="src/http/ngx_http.h \ src/http/ngx_http_script.h \ src/http/ngx_http_upstream.h \ src/http/ngx_http_upstream_round_robin.h \ - src/http/ngx_http_busy_lock.h" + src/http/ngx_http_busy_lock.h \ + src/http/ngx_http_probe.h" HTTP_SRCS="src/http/ngx_http.c \ src/http/ngx_http_core_module.c \ @@ -529,3 +537,8 @@ NGX_GOOGLE_PERFTOOLS_MODULE=ngx_google_perftools_module NGX_GOOGLE_PERFTOOLS_SRCS=src/misc/ngx_google_perftools_module.c NGX_CPP_TEST_SRCS=src/misc/ngx_cpp_test_module.cpp + +NGX_DTRACE_PROVIDERS=src/dtrace/nginx_provider.d + +NGX_TAPSET_SRCS=src/dtrace/nginx.stp + diff --git a/auto/summary b/auto/summary index dcebec9..3cb269e 100644 --- a/auto/summary +++ b/auto/summary @@ -92,6 +92,19 @@ else echo " nginx logs errors to stderr" fi +if [ $NGX_DTRACE = YES ]; then + cat << END + nginx dtrace static probes enabled +END + + if [ $DTRACE_FROM_SYSTEMTAP = YES ]; then + cat << END + nginx systemtap tapset prefix: "$NGX_TAPSET_PREFIX" + nginx systemtap wrapper script: "$NGX_STAP_NGX_PATH" +END + fi +fi + cat << END nginx http access log file: "$NGX_HTTP_LOG_PATH" nginx http client request body temporary files: "$NGX_HTTP_CLIENT_TEMP_PATH" diff --git a/configure b/configure index d7d8189..05fab87 100755 --- a/configure +++ b/configure @@ -23,6 +23,9 @@ if [ $NGX_DEBUG = YES ]; then have=NGX_DEBUG . auto/have fi +if [ $NGX_DTRACE = YES ]; then + have=NGX_DTRACE . auto/have +fi if test -z "$NGX_PLATFORM"; then echo "checking for OS" diff --git a/src/core/ngx_core_probe.h b/src/core/ngx_core_probe.h new file mode 100644 index 0000000..91bf91e --- /dev/null +++ b/src/core/ngx_core_probe.h @@ -0,0 +1,25 @@ +#ifndef _NGX_CORE_PROBE_H_INCLUDED_ +#define _NGX_CORE_PROBE_H_INCLUDED_ + + +#include +#include +#include + + +#if (NGX_DTRACE) + +#include +#include + +#define ngx_core_probe_create_pool_done(pool, size) \ + NGINX_CREATE_POOL_DONE(pool, size) + +#else /* !(NGX_DTRACE) */ + +#define ngx_core_probe_create_pool_done(pool, size) + +#endif + + +#endif /* _NGX_CORE_PROBE_H_INCLUDED_ */ diff --git a/src/core/ngx_palloc.c b/src/core/ngx_palloc.c index efbc244..8d81aab 100644 --- a/src/core/ngx_palloc.c +++ b/src/core/ngx_palloc.c @@ -7,6 +7,7 @@ #include #include +#include static void *ngx_palloc_block(ngx_pool_t *pool, size_t size); @@ -37,6 +38,8 @@ ngx_create_pool(size_t size, ngx_log_t *log) p->cleanup = NULL; p->log = log; + ngx_core_probe_create_pool_done(p, size); + return p; } diff --git a/src/dtrace/nginx.stp b/src/dtrace/nginx.stp new file mode 100644 index 0000000..e824daf --- /dev/null +++ b/src/dtrace/nginx.stp @@ -0,0 +1,299 @@ +/* tapset for nginx */ + + +function ngx_indent(n, delta) +{ + s = "" + for (i = 0; i < n; i++) { + s .= delta + } + + return s +} + + +function ngx_http_subreq_depth(r) +{ + depth = 0 + + for (pr = @cast(r, "ngx_http_request_t", "NGX_SBIN_PATH")->parent; + pr != 0; + pr = @cast(pr, "ngx_http_request_t", "NGX_SBIN_PATH")->parent) + { + depth++ + } + + return depth +} + + +function ngx_http_req_parent(r) +{ + return @cast(r, "ngx_http_request_s", "NGX_SBIN_PATH")->parent +} + + +/* retrieve the request uri string from the ngx_http_request_t pointer */ +function ngx_http_req_uri(r) +{ + len = @cast(r, "ngx_http_request_s", "NGX_SBIN_PATH")->uri->len + + if (len == 0) { + return "" + } + + return user_string_n(@cast(r, "ngx_http_request_s", "NGX_SBIN_PATH")->uri->data, len) +} + + +/* retrieve the request query string from the ngx_http_request_t pointer */ +function ngx_http_req_args(r) +{ + len = @cast(r, "ngx_http_request_s", "NGX_SBIN_PATH")->args->len + + if (len == 0) { + return "" + } + + return user_string_n(@cast(r, "ngx_http_request_s", "NGX_SBIN_PATH")->args->data, len) +} + + +/* retrieve the first command name (or directive name) from + * the ngx_module_t pointer */ +function ngx_http_module_cmd(m) +{ + cmds = @cast(m, "ngx_module_t", "NGX_SBIN_PATH")->commands + if (cmds == 0) { + return "" + } + + len = @cast(cmds, "ngx_command_t", "NGX_SBIN_PATH")->name->len + + if (len == 0) { + return "" + } + + return user_string_n(@cast(cmds, "ngx_command_t", "NGX_SBIN_PATH")->name->data, len) +} + + +function ngx_chain_buf(cl) +{ + return @cast(cl, "ngx_chain_t", "NGX_SBIN_PATH")->buf +} + + +function ngx_chain_next(cl) +{ + return @cast(cl, "ngx_chain_t", "NGX_SBIN_PATH")->next +} + +function ngx_buf_tag(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->tag +} + +function ngx_buf_in_memory(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->temporary + || @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->memory + || @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->mmap +} + + +function ngx_buf_pos(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->pos +} + + +function ngx_buf_file_pos(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->file_pos +} + + +function ngx_buf_last(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->last +} + + +function ngx_buf_file_last(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->file_last +} + + +function ngx_buf_end(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->end +} + + +function ngx_buf_in_file(b) +{ + return @cast(b, "ngx_buf_t", "NGX_SBIN_PATH")->in_file +} + + +function ngx_buf_last_buf(b) +{ + return @cast(b, "ngx_buf_t", "/home/agentzh/git/lua-nginx-module/work/nginx/sbin/nginx")->last_buf +} + + +function ngx_buf_last_in_chain(b) +{ + return @cast(b, "ngx_buf_t", "/home/agentzh/git/lua-nginx-module/work/nginx/sbin/nginx")->last_in_chain +} + + +function ngx_buf_sync(b) +{ + return @cast(b, "ngx_buf_t", "/home/agentzh/git/lua-nginx-module/work/nginx/sbin/nginx")->sync +} + + +function ngx_buf_flush(b) +{ + return @cast(b, "ngx_buf_t", "/home/agentzh/git/lua-nginx-module/work/nginx/sbin/nginx")->flush +} + + +function ngx_buf_size(b) +{ + if (ngx_buf_in_memory(b)) { + return ngx_buf_last(b) - ngx_buf_pos(b) + } + + return ngx_buf_file_last(b) - ngx_buf_file_pos(b) +} + + +function ngx_buf_data(b) +{ + return user_string_n(ngx_buf_pos(b), ngx_buf_last(b) - ngx_buf_pos(b)) +} + + +function ngx_chain_writer_ctx_out(ctx) +{ + return @cast(c, "ngx_chain_writer_ctx_t", "NGX_SBIN_PATH")->out +} + + +function ngx_chain_dump:string (input) +{ + if (input == 0) { + return "NULL" + } + + out = "" + cl = input + while (cl) { + buf = ngx_chain_buf(cl) + + if (ngx_buf_in_memory(buf)) { + out .= sprintf("[%s]", text_str(ngx_buf_data(buf))) + + } else { + out .= "\"\"" + } + + if (ngx_buf_in_file(buf)) { + out .= sprintf("", ngx_buf_file_pos(buf), + ngx_buf_file_last(buf)) + } + + if (ngx_buf_last_buf(buf)) { + out .= "" + } + + if (ngx_buf_last_in_chain(buf)) { + out .= "" + } + + if (ngx_buf_sync(buf)) { + out .= "" + } + + if (ngx_buf_flush(buf)) { + out .= "" + } + + tag = ngx_buf_tag(buf) + if (tag) { + out .= sprintf("", tag) + } + + cl = ngx_chain_next(cl) + if (cl) { + out .= " " + } + } + return out +} + + +function ngx_pool_cleanup_file_name(c) +{ + return user_string(@cast(c, "ngx_pool_cleanup_file_t", "NGX_SBIN_PATH")->name) +} + + +function ngx_http_req_content_length(r) +{ + return @cast(r, "ngx_http_request_t", "NGX_SBIN_PATH")->headers_in->content_length_n +} + + +function ngx_http_req_body_temp_file_name(r) +{ + rb = @cast(r, "ngx_http_request_t", "NGX_SBIN_PATH")->request_body + if (!rb) { + return "" + } + + tf = @cast(rb, "ngx_http_request_body_t", "NGX_SBIN_PATH")->temp_file + if (!tf) { + return "" + } + + len = @cast(tf, "ngx_temp_file_t", "NGX_SBIN_PATH")->file->name->len + + return user_string_n(@cast(tf, "ngx_temp_file_t", "NGX_SBIN_PATH")->file->name->data, len) +} + + +function ngx_table_elt_key(e) +{ + len = @cast(e, "ngx_table_elt_t", "NGX_SBIN_PATH")->key->len + + return user_string_n(@cast(e, "ngx_table_elt_t", "NGX_SBIN_PATH")->key->data, len) +} + + +function ngx_table_elt_value(e) +{ + len = @cast(e, "ngx_table_elt_t", "NGX_SBIN_PATH")->value->len + + return user_string_n(@cast(e, "ngx_table_elt_t", "NGX_SBIN_PATH")->value->data, len) +} + + +function ngx_iovec_dump:string (iov, iovcnt) { + out = "" + for (i = 0; i < iovcnt; i++) { + out .= sprintf("\"%s\"(%p)", text_str(user_string_n( + @cast(iov, "struct iovec")[i]->iov_base, + @cast(iov, "struct iovec")[i]->iov_len) + ), @cast(iov, "struct iovec")[i]->iov_base) + if (i != iovcnt - 1) { + out .= " " + } + } + return out +} + diff --git a/src/dtrace/nginx_provider.d b/src/dtrace/nginx_provider.d new file mode 100644 index 0000000..147ca12 --- /dev/null +++ b/src/dtrace/nginx_provider.d @@ -0,0 +1,41 @@ +typedef struct { int dummy; } ngx_http_request_t; +typedef struct { int dummy; } ngx_str_t; +typedef int64_t ngx_int_t; +typedef uint64_t ngx_uint_t; +typedef ngx_uint_t ngx_msec_t; +typedef struct { int dummy; } ngx_module_t; +typedef struct { int dummy; } ngx_http_module_t; +typedef struct { int dummy; } ngx_table_elt_t; +typedef struct { int dummy; } ngx_event_t; +typedef struct { int dummy; } ngx_pool_t; +typedef char unsigned u_char; + + +provider nginx { + /* probes for subrequests */ + probe http__subrequest__cycle(ngx_http_request_t *pr, ngx_str_t *uri, ngx_str_t *args); + probe http__subrequest__start(ngx_http_request_t *r); + probe http__subrequest__finalize_writing(ngx_http_request_t *r); + probe http__subrequest__finalize_nonactive(ngx_http_request_t *r); + probe http__subrequest__wake__parent(ngx_http_request_t *r); + probe http__subrequest__done(ngx_http_request_t *r); + probe http__subrequest__post__start(ngx_http_request_t *r, ngx_int_t rc); + probe http__subrequest__post__done(ngx_http_request_t *r, ngx_int_t rc); + probe http__module__post__config(ngx_module_t *m); + probe http__read__body__abort(ngx_http_request_t *r, char *reason); + probe http__read__body__done(ngx_http_request_t *r); + probe http__read__req__line__done(ngx_http_request_t *r); + probe http__read__req__header__done(ngx_http_request_t *r, ngx_table_elt_t *h); + probe timer__add(ngx_event_t *ev, ngx_msec_t timer); + probe timer__del(ngx_event_t *ev); + probe timer__expire(ngx_event_t *ev); + probe create__pool__done(ngx_pool_t *pool, size_t size); +}; + + +#pragma D attributes Evolving/Evolving/Common provider nginx provider +#pragma D attributes Private/Private/Unknown provider nginx module +#pragma D attributes Private/Private/Unknown provider nginx function +#pragma D attributes Private/Private/Common provider nginx name +#pragma D attributes Evolving/Evolving/Common provider nginx args + diff --git a/src/dtrace/stap-nginx b/src/dtrace/stap-nginx new file mode 100755 index 0000000..1bca4cf --- /dev/null +++ b/src/dtrace/stap-nginx @@ -0,0 +1,6 @@ +#!/bin/sh + +PATH="NGX_SBIN_DIR:$PATH" +export PATH +exec stap -d "NGX_SBIN_PATH" -I "NGX_TAPSET_PREFIX" "$@" + diff --git a/src/event/ngx_event_probe.h b/src/event/ngx_event_probe.h new file mode 100644 index 0000000..5aa0397 --- /dev/null +++ b/src/event/ngx_event_probe.h @@ -0,0 +1,33 @@ +#ifndef _NGX_EVENT_PROBE_H_INCLUDED_ +#define _NGX_EVENT_PROBE_H_INCLUDED_ + + +#include +#include +#include + + +#if (NGX_DTRACE) + +#include +#include + +#define ngx_event_probe_timer_add(ev, timer) \ + NGINX_TIMER_ADD(ev, timer) + +#define ngx_event_probe_timer_del(ev) \ + NGINX_TIMER_DEL(ev) + +#define ngx_event_probe_timer_expire(ev) \ + NGINX_TIMER_EXPIRE(ev) + +#else /* !(NGX_DTRACE) */ + +#define ngx_event_probe_timer_add(ev, timer) +#define ngx_event_probe_timer_del(ev) +#define ngx_event_probe_timer_expire(ev) + +#endif + + +#endif /* _NGX_EVENT_PROBE_H_INCLUDED_ */ diff --git a/src/event/ngx_event_timer.c b/src/event/ngx_event_timer.c index 177ac1c..531cccc 100644 --- a/src/event/ngx_event_timer.c +++ b/src/event/ngx_event_timer.c @@ -144,6 +144,8 @@ ngx_event_expire_timers(void) } #endif + ngx_event_probe_timer_expire(ev); + ev->timedout = 1; ev->handler(ev); diff --git a/src/event/ngx_event_timer.h b/src/event/ngx_event_timer.h index ec9b316..6095d8c 100644 --- a/src/event/ngx_event_timer.h +++ b/src/event/ngx_event_timer.h @@ -12,6 +12,7 @@ #include #include #include +#include #define NGX_TIMER_INFINITE (ngx_msec_t) -1 @@ -35,6 +36,8 @@ extern ngx_thread_volatile ngx_rbtree_t ngx_event_timer_rbtree; static ngx_inline void ngx_event_del_timer(ngx_event_t *ev) { + ngx_event_probe_timer_del(ev); + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "event timer del: %d: %M", ngx_event_ident(ev->data), ev->timer.key); @@ -85,6 +88,8 @@ ngx_event_add_timer(ngx_event_t *ev, ngx_msec_t timer) ev->timer.key = key; + ngx_event_probe_timer_add(ev, timer); + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, "event timer add: %d: %M:%M", ngx_event_ident(ev->data), timer, ev->timer.key); diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c index f1f8a48..91bdd64 100644 --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -8,6 +8,7 @@ #include #include #include +#include static char *ngx_http_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); @@ -307,6 +308,9 @@ ngx_http_block(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) module = ngx_modules[m]->ctx; if (module->postconfiguration) { + + ngx_http_probe_module_post_config(ngx_modules[m]); + if (module->postconfiguration(cf) != NGX_OK) { return NGX_CONF_ERROR; } diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index e02a251..6371fbc 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -8,6 +8,7 @@ #include #include #include +#include typedef struct { @@ -2377,6 +2378,8 @@ ngx_http_subrequest(ngx_http_request_t *r, r->main->subrequests--; if (r->main->subrequests == 0) { + ngx_http_probe_subrequest_cycle(r, uri, args); + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "subrequests cycle while processing \"%V\"", uri); r->main->subrequests = 1; @@ -2489,6 +2492,8 @@ ngx_http_subrequest(ngx_http_request_t *r, *psr = sr; + ngx_http_probe_subrequest_start(sr); + return ngx_http_post_request(sr, NULL); } diff --git a/src/http/ngx_http_probe.h b/src/http/ngx_http_probe.h new file mode 100644 index 0000000..d7d2d45 --- /dev/null +++ b/src/http/ngx_http_probe.h @@ -0,0 +1,75 @@ +#ifndef _NGX_HTTP_PROBE_H_INCLUDED_ +#define _NGX_HTTP_PROBE_H_INCLUDED_ + + +#include +#include +#include + + +#if (NGX_DTRACE) + +#include + +#define ngx_http_probe_subrequest_cycle(pr, uri, args) \ + NGINX_HTTP_SUBREQUEST_CYCLE(pr, uri, args) + +#define ngx_http_probe_subrequest_start(r) \ + NGINX_HTTP_SUBREQUEST_START(r) + +#define ngx_http_probe_subrequest_finalize_writing(r) \ + NGINX_HTTP_SUBREQUEST_FINALIZE_WRITING(r) + +#define ngx_http_probe_subrequest_finalize_nonactive(r) \ + NGINX_HTTP_SUBREQUEST_FINALIZE_NONACTIVE(r) + +#define ngx_http_probe_subrequest_finalize_nonactive(r) \ + NGINX_HTTP_SUBREQUEST_FINALIZE_NONACTIVE(r) + +#define ngx_http_probe_subrequest_wake_parent(r) \ + NGINX_HTTP_SUBREQUEST_WAKE_PARENT(r) + +#define ngx_http_probe_subrequest_done(r) \ + NGINX_HTTP_SUBREQUEST_DONE(r) + +#define ngx_http_probe_subrequest_post_start(r, rc) \ + NGINX_HTTP_SUBREQUEST_POST_START(r, rc) + +#define ngx_http_probe_subrequest_post_done(r, rc) \ + NGINX_HTTP_SUBREQUEST_POST_DONE(r, rc) + +#define ngx_http_probe_module_post_config(m) \ + NGINX_HTTP_MODULE_POST_CONFIG(m) + +#define ngx_http_probe_read_body_abort(r, reason) \ + NGINX_HTTP_READ_BODY_ABORT(r, reason) + +#define ngx_http_probe_read_body_done(r) \ + NGINX_HTTP_READ_BODY_DONE(r) + +#define ngx_http_probe_read_req_line_done(r) \ + NGINX_HTTP_READ_REQ_LINE_DONE(r) + +#define ngx_http_probe_read_req_header_done(r, h) \ + NGINX_HTTP_READ_REQ_HEADER_DONE(r, h) + +#else /* !(NGX_DTRACE) */ + +#define ngx_http_probe_subrequest_cycle(pr, uri, args) +#define ngx_http_probe_subrequest_start(r) +#define ngx_http_probe_subrequest_finalize_writing(r) +#define ngx_http_probe_subrequest_finalize_nonactive(r) +#define ngx_http_probe_subrequest_wake_parent(r) +#define ngx_http_probe_subrequest_done(r) +#define ngx_http_probe_subrequest_post_start(r, rc) +#define ngx_http_probe_subrequest_post_done(r, rc) +#define ngx_http_probe_module_post_config(m) +#define ngx_http_probe_read_body_abort(r, reason) +#define ngx_http_probe_read_body_done(r) +#define ngx_http_probe_read_req_line_done(r) +#define ngx_http_probe_read_req_header_done(r, h) + +#endif /* NGX_DTRACE */ + + +#endif /* _NGX_HTTP_PROBE_H_INCLUDED_ */ diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index ee00fd3..441b5cc 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -8,6 +8,7 @@ #include #include #include +#include static void ngx_http_init_request(ngx_event_t *ev); @@ -861,6 +862,8 @@ ngx_http_process_request_line(ngx_event_t *rev) } #endif + ngx_http_probe_read_req_line_done(r); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "http request line: \"%V\"", &r->request_line); @@ -1108,6 +1111,8 @@ ngx_http_process_request_headers(ngx_event_t *rev) return; } + ngx_http_probe_read_req_header_done(r, h); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "http header: \"%V: %V\"", &h->key, &h->value); @@ -1958,7 +1963,11 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) } if (r != r->main && r->post_subrequest) { + ngx_http_probe_subrequest_post_start(r, rc); + rc = r->post_subrequest->handler(r, r->post_subrequest->data, rc); + + ngx_http_probe_subrequest_post_done(r, rc); } if (rc == NGX_ERROR @@ -2008,6 +2017,8 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) if (r->buffered || r->postponed) { + ngx_http_probe_subrequest_finalize_writing(r); + if (ngx_http_set_write_handler(r) != NGX_OK) { ngx_http_terminate_request(r, 0); } @@ -2044,10 +2055,14 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) pr->postponed = pr->postponed->next; } + ngx_http_probe_subrequest_done(r); + c->data = pr; } else { + ngx_http_probe_subrequest_finalize_nonactive(r); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "http finalize non-active request: \"%V?%V\"", &r->uri, &r->args); @@ -2059,6 +2074,8 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) } } + ngx_http_probe_subrequest_wake_parent(r); + if (ngx_http_post_request(pr, NULL) != NGX_OK) { r->main->count++; ngx_http_terminate_request(r, 0); diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c index 3c69d05..89858d0 100644 --- a/src/http/ngx_http_request_body.c +++ b/src/http/ngx_http_request_body.c @@ -8,6 +8,7 @@ #include #include #include +#include static void ngx_http_read_client_request_body_handler(ngx_http_request_t *r); @@ -40,11 +41,15 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, r->main->count++; if (r->request_body || r->discard_body) { + ngx_http_probe_read_body_abort(r, + r->request_body ? "body exists" + : "body discarded"); post_handler(r); return NGX_OK; } if (ngx_http_test_expect(r) != NGX_OK) { + ngx_http_probe_read_body_abort(r, "test expect failed"); rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } @@ -58,6 +63,7 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, r->request_body = rb; if (r->headers_in.content_length_n < 0) { + ngx_http_probe_read_body_abort(r, "no content length"); post_handler(r); return NGX_OK; } @@ -68,11 +74,14 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, if (r->request_body_in_file_only) { if (ngx_http_write_request_body(r, NULL) != NGX_OK) { + ngx_http_probe_read_body_abort(r, "create temp file failed"); rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } } + ngx_http_probe_read_body_done(r); + post_handler(r); return NGX_OK; @@ -130,11 +139,14 @@ ngx_http_read_client_request_body(ngx_http_request_t *r, if (r->request_body_in_file_only) { if (ngx_http_write_request_body(r, rb->bufs) != NGX_OK) { + ngx_http_probe_read_body_abort(r, "write temp file failed"); rc = NGX_HTTP_INTERNAL_SERVER_ERROR; goto done; } } + ngx_http_probe_read_body_done(r); + post_handler(r); return NGX_OK; @@ -239,6 +251,7 @@ ngx_http_read_client_request_body_handler(ngx_http_request_t *r) ngx_int_t rc; if (r->connection->read->timedout) { + ngx_http_probe_read_body_abort(r, "timed out"); r->connection->timedout = 1; ngx_http_finalize_request(r, NGX_HTTP_REQUEST_TIME_OUT); return; @@ -273,6 +286,7 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) if (rb->buf->last == rb->buf->end) { if (ngx_http_write_request_body(r, rb->to_write) != NGX_OK) { + ngx_http_probe_read_body_abort(r, "write temp file failed"); return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -296,6 +310,9 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) } if (n == 0) { + + ngx_http_probe_read_body_abort(r, "connection closed"); + ngx_log_error(NGX_LOG_INFO, c->log, 0, "client prematurely closed connection"); } @@ -346,6 +363,7 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) /* save the last part */ if (ngx_http_write_request_body(r, rb->to_write) != NGX_OK) { + ngx_http_probe_read_body_abort(r, "write temp file failed"); return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -375,6 +393,8 @@ ngx_http_do_read_client_request_body(ngx_http_request_t *r) r->read_event_handler = ngx_http_block_reading; + ngx_http_probe_read_body_done(r); + rb->post_handler(r); return NGX_OK;