erl_tracer

Erlang Tracer Behaviour

A behaviour module for implementing the back end of the erlang tracing system. The functions in this module will be called whenever a trace probe is triggered. Both the enabled and trace functions are called in the context of the entity that triggered the trace probe. This means that the overhead by having the tracing enabled will be greatly effected by how much time is spent in these functions. So do as little work as possible in these functions.

Note!

All functions in this behaviour have to be implemented as NIF's. This is a limitation that may the lifted in the future. There is an example tracer module nif implementation at the end of this page.

Warning!

Do not send messages or issue port commands to the Tracee in any of the callbacks. Doing so is not allowed and can cause all sorts of strange behaviour, including but not limited to infinite recursions.

Types


trace_tag_send() = send | send_to_non_existing_process

trace_tag_receive() = 'receive'

trace_tag_call() =
            call | return_to | return_from | exception_from

trace_tag_procs() =
            spawn |
            spawned |
            exit |
            link |
            unlink |
            getting_linked |
            getting_unlinked |
            register |
            unregister

trace_tag_ports() =
            open |
            closed |
            link |
            unlink |
            getting_linked |
            getting_unlinked

trace_tag_running_procs() =
            in | out | in_exiting | out_exiting | out_exited

trace_tag_running_ports() =
            in | out | in_exiting | out_exiting | out_exited

trace_tag_gc() =
            gc_minor_start | gc_minor_end | gc_major_start | gc_major_end

trace_tag() =
            trace_tag_send() |
            trace_tag_receive() |
            trace_tag_call() |
            trace_tag_procs() |
            trace_tag_ports() |
            trace_tag_running_procs() |
            trace_tag_running_ports() |
            trace_tag_gc()

The different trace tags that the tracer will be called with. Each trace tag is described in greater detail in Module:trace/6

tracee() = port() | pid() | undefined

The process or port that the trace belongs to.

trace_opts() =
            #{match_spec_result => true | term(),
              scheduler_id => undefined | integer() >= 0,
              timestamp =>
                  undefined |
                  timestamp |
                  cpu_timestamp |
                  monotonic |
                  strict_monotonic}

The options for the tracee.

timestamp
If not set to undefined, the tracer has been requested to include a timestamp.
match_spec_result
If not set to true, the tracer has been requested to include the output of a match specification that was run.
scheduler_id
Set to a number if the scheduler id is to be included by the tracer. Otherwise it is set to undefined.

tracer_state() = term()

The state which is given when calling erlang:trace(PidPortSpec,true,[{tracer,Module,TracerState}]). The tracer state is an immutable value that is passed to erl_tracer callbacks and should contain all the data that is needed to generate the trace event.

CALLBACK FUNCTIONS

The following functions should be exported from a erl_tracer callback module.

Module:enabled/3
Mandatory
Module:trace/6
Mandatory
Module:enabled_procs/3
Optional
Module:trace_procs/6
Optional
Module:enabled_ports/3
Optional
Module:trace_ports/6
Optional
Module:enabled_running_ports/3
Optional
Module:trace_running_ports/6
Optional
Module:enabled_running_procs/3
Optional
Module:trace_running_procs/6
Optional

Functions


Module:enabled(TraceTag, TracerState, Tracee) -> Result

  • TraceTag = trace_tag() | trace_status
  • TracerState = term()
  • Tracee = tracee()
  • Result = trace | discard | remove

This callback will be called whenever a tracepoint is triggered. It allows the tracer to decide whether a trace should be generated or not. This check is made as early as possible in order to limit the amount of overhead associated with tracing. If trace is returned the necessary trace data will be created and the trace call-back of the tracer will be called. If discard is returned, this trace call will be discarded and no call to trace will be done.

trace_status is a special type of TraceTag which is used to check if the tracer should still be active. It is called in multiple scenarios, but most significantly it is used when tracing is started using this tracer. If remove is returned when the trace_status is checked, the tracer will be removed from the tracee.

This function may be called multiple times per tracepoint, so it is important that it is both fast and side effect free.

Module:trace(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

  • TraceTag = trace_tag()
  • TracerState = term()
  • Tracee = tracee()
  • FirstTraceTerm = term()
  • SecondTraceTerm = term() | undefined
  • Opts = trace_opts()
  • Result = ok

This callback will be called when a tracepoint is triggered and the Module:enabled/3 callback returned trace. In it any side effects needed by the tracer should be done. The tracepoint payload is located in the FirstTraceTerm and SecondTraceTerm. The content of the TraceTerms depends on which TraceTag has been triggered. The FirstTraceTerm and SecondTraceTerm correspond to the fourth and fifth slot in the trace tuples described in erlang:trace/3. If the tuple only has four elements, SecondTraceTerm will be undefined.

Module:trace(seq_trace, TracerState, Label, SeqTraceInfo, undefined, Opts) -> Result
  • TracerState = term()
  • Label = term()
  • SeqTraceInfo = term()
  • Opts = trace_opts()
  • Result = ok

The TraceTag seq_trace is handled a little bit differently. There is not Tracee for seq_trace, instead the Label associated with the seq_trace event is given. For more info on what Label and SeqTraceInfo can be see the seq_trace manual.

Module:enabled_procs(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag procs is triggered.

If enabled_procs/3 is not defined enabled/3 will be called instead.

Module:trace_procs(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

  • TraceTag = trace_tag()
  • TracerState = term()
  • Tracee = tracee()
  • FirstTraceTerm = term()
  • SecondTraceTerm = term() | undefined
  • Opts = trace_opts()
  • Result = ok

This callback will be called when a tracepoint is triggered and the Module:enabled_procs/3 callback returned trace.

If trace_procs/6 is not defined trace/6 will be called instead.

Module:enabled_ports(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag ports is triggered.

If enabled_ports/3 is not defined enabled/3 will be called instead.

Module:trace_ports(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

  • TraceTag = trace_tag()
  • TracerState = term()
  • Tracee = tracee()
  • FirstTraceTerm = term()
  • SecondTraceTerm = term() | undefined
  • Opts = trace_opts()
  • Result = ok

This callback will be called when a tracepoint is triggered and the Module:enabled_ports/3 callback returned trace.

If trace_ports/6 is not defined trace/6 will be called instead.

Module:enabled_running_procs(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag running_procs | running is triggered.

If enabled_running_procs/3 is not defined enabled/3 will be called instead.

Module:trace_running_procs(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

This callback will be called when a tracepoint is triggered and the Module:enabled_running_procs/3 callback returned trace.

If trace_running_procs/6 is not defined trace/6 will be called instead.

Module:enabled_running_ports(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag running_ports is triggered.

If enabled_running_ports/3 is not defined enabled/3 will be called instead.

Module:trace_running_ports(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

This callback will be called when a tracepoint is triggered and the Module:enabled_running_ports/3 callback returned trace.

If trace_running_ports/6 is not defined trace/6 will be called instead.

Module:enabled_call(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag call | return_to is triggered.

If enabled_call/3 is not defined enabled/3 will be called instead.

Module:trace_call(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

This callback will be called when a tracepoint is triggered and the Module:enabled_call/3 callback returned trace.

If trace_call/6 is not defined trace/6 will be called instead.

Module:enabled_send(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag send is triggered.

If enabled_send/3 is not defined enabled/3 will be called instead.

Module:trace_send(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

This callback will be called when a tracepoint is triggered and the Module:enabled_send/3 callback returned trace.

If trace_send/6 is not defined trace/6 will be called instead.

Module:enabled_receive(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag 'receive' is triggered.

If enabled_receive/3 is not defined enabled/3 will be called instead.

Module:trace_receive(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

This callback will be called when a tracepoint is triggered and the Module:enabled_receive/3 callback returned trace.

If trace_receive/6 is not defined trace/6 will be called instead.

Module:enabled_garbage_collection(TraceTag, TracerState, Tracee) -> Result

This callback will be called whenever a tracepoint with trace flag garbage_collection is triggered.

If enabled_garbage_collection/3 is not defined enabled/3 will be called instead.

Module:trace_garbage_collection(TraceTag, TracerState, Tracee, FirstTraceTerm, SecondTraceTerm, Opts) -> Result

This callback will be called when a tracepoint is triggered and the Module:enabled_garbage_collection/3 callback returned trace.

If trace_garbage_collection/6 is not defined trace/6 will be called instead.

Erl Tracer Module example

In the example below a tracer module with a nif backend sends a message for each send trace tag containing only the sender and receiver. Using this tracer module, a much more lightweight message tracer is used that only records who sent messages to who.

Here is an example session using it on Linux.

$ gcc -I erts-8.0/include/ -fPIC -shared -o erl_msg_tracer.so erl_msg_tracer.c
$ erl
Erlang/OTP 19 [DEVELOPMENT] [erts-8.0] [source-ed2b56b] [64-bit] [smp:8:8] [async-threads:10] [hipe] [kernel-poll:false]

Eshell V8.0  (abort with ^G)
1> c(erl_msg_tracer), erl_msg_tracer:load().
ok
2> Tracer = spawn(fun F() -> receive M -> io:format("~p~n",[M]), F() end end).
<0.37.0>
3> erlang:trace(new, true, [send,{tracer, erl_msg_tracer, Tracer}]).
0
{<0.39.0>,<0.27.0>}
4> {ok, D} = file:open("/tmp/tmp.data",[write]).
{trace,#Port<0.486>,<0.40.0>}
{trace,<0.40.0>,<0.21.0>}
{trace,#Port<0.487>,<0.4.0>}
{trace,#Port<0.488>,<0.4.0>}
{trace,#Port<0.489>,<0.4.0>}
{trace,#Port<0.490>,<0.4.0>}
{ok,<0.40.0>}
{trace,<0.41.0>,<0.27.0>}
5>
    

erl_msg_tracer.erl

-module(erl_msg_tracer).

-export([enabled/3, trace/6, load/0]).

load() ->
    erlang:load_nif("erl_msg_tracer", []).

enabled(_, _, _) ->
    error.

trace(_, _, _,_, _, _) ->
    error.
    

erl_msg_tracer.c

#include "erl_nif.h"

/* NIF interface declarations */
static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info);
static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
static void unload(ErlNifEnv* env, void* priv_data);

/* The NIFs: */
static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);

static ErlNifFunc nif_funcs[] = {
    {"enabled", 3, enabled},
    {"trace", 6, trace}
};

ERL_NIF_INIT(erl_msg_tracer, nif_funcs, load, NULL, upgrade, unload)

static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
{
    *priv_data = NULL;
    return 0;
}

static void unload(ErlNifEnv* env, void* priv_data)
{

}

static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data,
		   ERL_NIF_TERM load_info)
{
    if (*old_priv_data != NULL || *priv_data != NULL) {
	return -1; /* Don't know how to do that */
    }
    if (load(env, priv_data, load_info)) {
	return -1;
    }
    return 0;
}

/*
 * argv[0]: TraceTag
 * argv[1]: TracerState
 * argv[2]: Tracee
 */
static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
    ErlNifPid to_pid;
    if (enif_get_local_pid(env, argv[1], &to_pid))
        if (!enif_is_process_alive(env, &to_pid))
            if (enif_is_identical(enif_make_atom(env, "trace_status"), argv[0]))
                /* tracer is dead so we should remove this tracepoint */
                return enif_make_atom(env, "remove");
            else
                return enif_make_atom(env, "discard");

    /* Only generate trace for when tracer != tracee */
    if (enif_is_identical(argv[1], argv[2]))
        return enif_make_atom(env, "discard");

    /* Only trigger trace messages on 'send' */
    if (enif_is_identical(enif_make_atom(env, "send"), argv[0]))
        return enif_make_atom(env, "trace");

    /* Have to answer trace_status */
    if (enif_is_identical(enif_make_atom(env, "trace_status"), argv[0]))
        return enif_make_atom(env, "trace");

    return enif_make_atom(env, "discard");
}

/*
 * argv[0]: TraceTag, should only be 'send'
 * argv[1]: TracerState, process to send {argv[2], argv[4]} to
 * argv[2]: Tracee
 * argv[3]: Message, ignored
 * argv[4]: Recipient
 * argv[5]: Options, ignored
 */
static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
{
    ErlNifPid to_pid;

    if (enif_get_local_pid(env, argv[1], &to_pid)) {
        ERL_NIF_TERM msg = enif_make_tuple3(env, enif_make_atom(env, "trace"), argv[2], argv[4]);
        enif_send(env, &to_pid, NULL, msg);
    }

    return enif_make_atom(env, "ok");
}