./ct_report/coverage/mongoose_instrument.COVER.html

1 -module(mongoose_instrument).
2
3 -behaviour(gen_server).
4
5 %% API
6 -export([config_spec/0,
7 start_link/0, persist/0,
8 set_up/1, set_up/3,
9 tear_down/1, tear_down/2,
10 span/4, span/5, span/6,
11 execute/3]).
12
13 %% Test API
14 -export([add_handler/2, remove_handler/1]).
15
16 %% gen_server callbacks
17 -export([init/1, handle_call/3, handle_cast/2, code_change/3, handle_info/2, terminate/2]).
18
19 -ignore_xref([start_link/0, set_up/3, tear_down/2, span/4, add_handler/2, remove_handler/1]).
20
21 -include("mongoose.hrl").
22 -include("mongoose_config_spec.hrl").
23
24 -type event_name() :: atom().
25 -type labels() :: #{host_type => mongooseim:host_type(),
26 function => atom()}. % to be extended
27 -type label_key() :: host_type | function. % to be extended
28 -type label_value() :: mongooseim:host_type() | atom(). % to be extended
29 -type metrics() :: #{metric_name() => metric_type()}.
30 -type metric_name() :: atom().
31 -type metric_type() :: gauge | spiral | histogram. % to be extended
32 -type measurements() :: #{atom() => term()}.
33 -type spec() :: {event_name(), labels(), config()}.
34 -type config() :: #{metrics => metrics(),
35 loglevel => logger:level(),
36 probe => probe_config()}.
37 -type probe_config() :: #{module := module(),
38 interval => pos_integer()}.
39 -type handler_key() :: atom(). % key in the `instrumentation' section of the config file
40 -type handler_fun() :: fun((event_name(), labels(), config(), measurements()) -> any()).
41 -type handlers() :: {[handler_fun()], config()}.
42 -type execution_time() :: integer().
43 -type measure_fun(Result) :: fun((execution_time(), Result) -> measurements()).
44
45 -callback config_spec() -> mongoose_config_spec:config_section().
46 -callback start() -> ok.
47 -callback stop() -> ok.
48 -callback set_up(event_name(), labels(), config()) -> boolean().
49 -callback handle_event(event_name(), labels(), config(), measurements()) -> any().
50
51 -optional_callbacks([config_spec/0, start/0, stop/0]).
52
53 -export_type([event_name/0, labels/0, label_key/0, label_value/0, config/0, measurements/0,
54 spec/0, handlers/0, metric_name/0, metric_type/0, probe_config/0]).
55
56 %% API
57
58 %% @doc Specifies the `instrumentation' section of the config file
59 -spec config_spec() -> mongoose_config_spec:config_section().
60 config_spec() ->
61 93 Items = [{atom_to_binary(Key), config_spec(Key)} || Key <- all_handler_keys()],
62 93 Options = #{<<"probe_interval">> => #option{type = integer, validate = positive}},
63 93 #section{items = maps:merge(maps:from_list(Items), Options),
64 defaults = #{<<"probe_interval">> => 15},
65 wrap = global_config,
66 include = always}.
67
68 -spec start_link() -> gen_server:start_ret().
69 start_link() ->
70 93 gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
71
72 %% @doc Saves the state to a persistent term, improving performance of `execute' and `span'.
73 %% On the other hand, future calls to `set_up' or `tear_down' will update the persistent term,
74 %% which makes them less performant.
75 %% You should call this function only once - after the initial setup, but before handling any load.
76 -spec persist() -> ok.
77 persist() ->
78 93 gen_server:call(?MODULE, persist).
79
80 %% @doc Sets up instrumentation for multiple events.
81 %% @see set_up/3
82 -spec set_up([spec()]) -> ok.
83 set_up(Specs) ->
84 3295 lists:foreach(fun({EventName, Labels, Config}) -> set_up(EventName, Labels, Config) end, Specs).
85
86 %% @doc Tears down instrumentation for multiple events.
87 %% @see tear_down/2
88 -spec tear_down([spec()]) -> ok.
89 tear_down(Specs) ->
90 1412 lists:foreach(fun({EventName, Labels, _Config}) -> tear_down(EventName, Labels) end, Specs).
91
92 %% @doc Sets up instrumentation for an event identified by `EventName' and `Labels'
93 %% according to `Config'. Fails if the event is already registered, or if the keys of `Labels'
94 %% are different than for already registered events with `EventName'.
95 -spec set_up(event_name(), labels(), config()) -> ok.
96 set_up(EventName, Labels, Config) ->
97 26186 case gen_server:call(?MODULE, {set_up, EventName, Labels, Config}) of
98 25977 ok -> ok;
99 209 {error, ErrorMap} -> error(ErrorMap)
100 end.
101
102 %% @doc Tears down instrumentation for an event identified by `EventName' and `Labels'.
103 %% This operation is idempotent.
104 -spec tear_down(event_name(), labels()) -> ok.
105 tear_down(EventName, Labels) ->
106 17451 gen_server:call(?MODULE, {tear_down, EventName, Labels}).
107
108 %% @doc Calls `F', measuring its result with `MeasureF', and calls attached event handlers.
109 %% @see span/5
110 -spec span(event_name(), labels(), fun(() -> Result), measure_fun(Result)) -> Result.
111 span(EventName, Labels, F, MeasureF) ->
112 
:-(
 span(EventName, Labels, F, [], MeasureF).
113
114 %% @doc Calls `F' with `Args', measuring its execution time.
115 %% The time and the result are passed to `MeasureF', which returns measurements.
116 %% The measurements are then passed to all handlers attached to
117 %% the event identified by `EventName' and `Labels'.
118 %% Fails without calling `F' if the event is not registered.
119 -spec span(event_name(), labels(), fun((...) -> Result), list(), measure_fun(Result)) -> Result.
120 span(EventName, Labels, F, Args, MeasureF) ->
121 13822 Handlers = get_handlers(EventName, Labels),
122 13822 {Time, Result} = timer:tc(F, Args),
123 13822 handle_event(EventName, Labels, MeasureF(Time, Result), Handlers),
124 13822 Result.
125
126 %% @doc Calls `Mod:F' with `Args', measuring its execution time.
127 %% @see span/5
128 -spec span(event_name(), labels(), module(), atom(), list(), measure_fun(Result)) -> Result.
129 span(EventName, Labels, Mod, F, Args, MeasureF) ->
130 208842 Handlers = get_handlers(EventName, Labels),
131 208842 {Time, Result} = timer:tc(Mod, F, Args),
132 208842 handle_event(EventName, Labels, MeasureF(Time, Result), Handlers),
133 208842 Result.
134
135 %% @doc Executes all handlers attached to the event identified by `EventName' and `Labels',
136 %% passing `Measurements' to them. Fails if the event is not registered.
137 -spec execute(event_name(), labels(), measurements()) -> ok.
138 execute(EventName, Labels, Measurements) ->
139 155679 Handlers = get_handlers(EventName, Labels),
140 155679 handle_event(EventName, Labels, Measurements, Handlers).
141
142 %% Test API
143
144 -spec add_handler(handler_key(), mongoose_config:value()) -> ok.
145 add_handler(Key, ConfigVal) ->
146 7 case gen_server:call(?MODULE, {add_handler, Key, ConfigVal}) of
147 7 ok -> ok;
148 
:-(
 {error, ErrorMap} -> error(ErrorMap)
149 end.
150
151 -spec remove_handler(handler_key()) -> ok.
152 remove_handler(Key) ->
153 7 case gen_server:call(?MODULE, {remove_handler, Key}) of
154 7 ok -> ok;
155 
:-(
 {error, ErrorMap} -> error(ErrorMap)
156 end.
157
158 %% gen_server callbacks
159
160 -type state() :: #{events := event_map(), probe_timers := probe_timer_map()}.
161 -type event_map() :: #{event_name() => #{labels() => handlers()}}.
162 -type probe_timer_map() :: #{{event_name(), labels()} => timer:tref()}.
163
164 -spec init([]) -> {ok, state()}.
165 init([]) ->
166 93 lists:foreach(fun start_handler/1, handler_modules()),
167 93 erlang:process_flag(trap_exit, true), % Make sure that terminate is called
168 93 persistent_term:erase(?MODULE), % Prevent inconsistency when restarted after a kill
169 93 {ok, #{events => #{}, probe_timers => #{}}}.
170
171 -spec handle_call(any(), gen_server:from(), state()) ->
172 {reply, ok | {ok, handlers()} | {error, map()}, state()}.
173 handle_call({set_up, EventName, Labels, Config}, _From,
174 #{events := Events, probe_timers := ProbeTimers} = State) ->
175 26186 case set_up_and_register_event(EventName, Labels, Config, Events) of
176 {error, _} = Error ->
177 209 {reply, Error, State};
178 NewEvents = #{} ->
179 25977 update_if_persisted(Events, NewEvents),
180 25977 NewProbeTimers = start_probe_if_needed(EventName, Labels, Config, ProbeTimers),
181 25977 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}}
182 end;
183 handle_call({tear_down, EventName, Labels}, _From,
184 #{events := Events, probe_timers := ProbeTimers}) ->
185 17451 NewProbeTimers = deregister_probe_timer(EventName, Labels, ProbeTimers),
186 17451 NewEvents = deregister_event(EventName, Labels, Events),
187 17451 update_if_persisted(Events, NewEvents),
188 17451 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}};
189 handle_call({add_handler, Key, ConfigOpts}, _From, State = #{events := Events}) ->
190 7 case mongoose_config:lookup_opt([instrumentation, Key]) of
191 {error, not_found} ->
192 7 mongoose_config:set_opt([instrumentation, Key], ConfigOpts),
193 7 Module = handler_module(Key),
194 7 start_handler(Module),
195 7 NewEvents = update_handlers(Events, [], [Module]),
196 7 update_if_persisted(Events, NewEvents),
197 7 {reply, ok, State#{events := NewEvents}};
198 {ok, ExistingConfig} ->
199 
:-(
 {reply, {error, #{what => handler_already_configured, handler_key => Key,
200 existing_config => ExistingConfig}},
201 State}
202 end;
203 handle_call({remove_handler, Key}, _From, State = #{events := Events}) ->
204 7 case mongoose_config:lookup_opt([instrumentation, Key]) of
205 {error, not_found} ->
206 
:-(
 {reply, {error, #{what => handler_not_configured, handler_key => Key}}, State};
207 {ok, _} ->
208 7 mongoose_config:unset_opt([instrumentation, Key]),
209 7 Module = handler_module(Key),
210 7 NewEvents = update_handlers(Events, [Module], []),
211 7 update_if_persisted(Events, NewEvents),
212 7 stop_handler(Module),
213 7 {reply, ok, State#{events := NewEvents}}
214 end;
215 handle_call(persist, _From, State = #{events := Events}) ->
216 93 persistent_term:put(?MODULE, Events),
217 93 {reply, ok, State};
218 handle_call({lookup, EventName, Labels}, _From, State = #{events := Events}) ->
219 648 {reply, lookup(EventName, Labels, Events), State};
220 handle_call(Request, From, State) ->
221 
:-(
 ?UNEXPECTED_CALL(Request, From),
222 
:-(
 {reply, {error, #{what => unexpected_call, request => Request}}, State}.
223
224 -spec handle_cast(any(), state()) -> {noreply, state()}.
225 handle_cast(Msg, State) ->
226 
:-(
 ?UNEXPECTED_CAST(Msg),
227 
:-(
 {noreply, State}.
228
229 -spec handle_info(any(), state()) -> {noreply, state()}.
230 handle_info(Info, State) ->
231 
:-(
 ?UNEXPECTED_INFO(Info),
232 
:-(
 {noreply, State}.
233
234 -spec terminate(any(), state()) -> ok.
235 terminate(_Reason, _State) ->
236 93 persistent_term:erase(?MODULE),
237 93 lists:foreach(fun stop_handler/1, handler_modules()).
238
239 -spec code_change(any(), state(), any()) -> {ok, state()}.
240 code_change(_OldVsn, State, _Extra) ->
241 
:-(
 {ok, State}.
242
243 %% Internal functions
244
245 -spec update_if_persisted(event_map(), event_map()) -> ok.
246 update_if_persisted(Events, NewEvents) ->
247 43442 try persistent_term:get(?MODULE) of
248 21113 Events -> persistent_term:put(?MODULE, NewEvents)
249 catch
250 22329 error:badarg -> ok
251 end.
252
253 -spec set_up_and_register_event(event_name(), labels(), config(), event_map()) ->
254 event_map() | {error, map()}.
255 set_up_and_register_event(EventName, Labels, Config, Events) ->
256 26186 LabelKeys = label_keys(Labels),
257 26186 case Events of
258 #{EventName := #{Labels := _}} ->
259 209 {error, #{what => event_already_registered,
260 event_name => EventName, labels => Labels}};
261 #{EventName := HandlerMap} ->
262 17900 {ExistingLabels, _, _} = maps:next(maps:iterator(HandlerMap)),
263 17900 case label_keys(ExistingLabels) of
264 LabelKeys ->
265 17900 Handlers = do_set_up(EventName, Labels, Config),
266 17900 Events#{EventName := HandlerMap#{Labels => Handlers}};
267 ExistingKeys ->
268 
:-(
 {error, #{what => inconsistent_labels,
269 event_name => EventName, labels => Labels,
270 existing_label_keys => ExistingKeys}}
271 end;
272 #{} ->
273 8077 Handlers = do_set_up(EventName, Labels, Config),
274 8077 Events#{EventName => #{Labels => Handlers}}
275 end.
276
277 -spec do_set_up(event_name(), labels(), config()) -> handlers().
278 do_set_up(EventName, Labels, Config) ->
279 25977 HandlerFuns = set_up_handlers(EventName, Labels, Config, handler_modules()),
280 25977 {HandlerFuns, Config}.
281
282 -spec start_probe_if_needed(event_name(), labels(), config(), probe_timer_map()) ->
283 probe_timer_map().
284 start_probe_if_needed(EventName, Labels, #{probe := ProbeConfig}, ProbeTimers) ->
285 29 TRef = mongoose_instrument_probe:start_probe_timer(EventName, Labels, ProbeConfig),
286 29 add_probe_timer(EventName, Labels, TRef, ProbeTimers);
287 start_probe_if_needed(_EventName, _Labels, _Config, ProbeTimers) ->
288 25948 ProbeTimers.
289
290 -spec add_probe_timer(event_name(), labels(), timer:tref(), probe_timer_map()) -> probe_timer_map().
291 add_probe_timer(EventName, Labels, TRef, ProbeTimers) ->
292 29 false = maps:is_key({EventName, Labels}, ProbeTimers), % sanity check to detect timer leak
293 29 ProbeTimers#{{EventName, Labels} => TRef}.
294
295 -spec update_handlers(event_map(), [module()], [module()]) -> event_map().
296 update_handlers(Events, ToRemove, ToAdd) ->
297 14 maps:map(fun(EventName, HandlerMap) ->
298 831 maps:map(fun(Labels, Handlers) ->
299 4083 update_event_handlers(EventName, Labels, Handlers,
300 ToRemove, ToAdd)
301 end, HandlerMap)
302 end, Events).
303
304 -spec update_event_handlers(event_name(), labels(), handlers(), [module()], [module()]) ->
305 handlers().
306 update_event_handlers(EventName, Labels, {HandlerFuns, Config}, ToRemove, ToAdd) ->
307 4083 FunsToRemove = modules_to_funs(ToRemove),
308 4083 FunsToAdd = set_up_handlers(EventName, Labels, Config, ToAdd),
309 4083 HandlerFuns = HandlerFuns -- FunsToAdd, % sanity check to prevent duplicates
310 4083 {(HandlerFuns -- FunsToRemove) ++ FunsToAdd, Config}.
311
312 -spec set_up_handlers(event_name(), labels(), config(), [module()]) -> [handler_fun()].
313 set_up_handlers(EventName, Labels, Config, Modules) ->
314 30060 UsedModules = lists:filter(fun(Mod) -> Mod:set_up(EventName, Labels, Config) end, Modules),
315 30060 modules_to_funs(UsedModules).
316
317 -spec deregister_event(event_name(), labels(), event_map()) -> event_map().
318 deregister_event(EventName, Labels, Events) ->
319 17451 case Events of
320 #{EventName := HandlerMap} ->
321 16893 case maps:remove(Labels, HandlerMap) of
322 Empty when Empty =:= #{} ->
323 6625 maps:remove(EventName, Events);
324 NewHandlerMap ->
325 10268 Events#{EventName := NewHandlerMap}
326 end;
327 #{} ->
328 558 Events
329 end.
330
331 -spec deregister_probe_timer(event_name(), labels(), probe_timer_map()) -> probe_timer_map().
332 deregister_probe_timer(EventName, Labels, ProbeTimers) ->
333 17451 case maps:take({EventName, Labels}, ProbeTimers) of
334 {TRef, NewProbeTimers} ->
335 29 timer:cancel(TRef),
336 29 NewProbeTimers;
337 error ->
338 17422 ProbeTimers % no timer was registered
339 end.
340
341 -spec lookup(event_name(), labels()) -> {ok, handlers()} | {error, map()}.
342 lookup(EventName, Labels) ->
343 378343 try persistent_term:get(?MODULE) of
344 Events ->
345 377695 lookup(EventName, Labels, Events)
346 catch
347 %% Although persist/0 should be called before handling traffic,
348 %% some instrumented events might happen before that, and they shouldn't fail.
349 error:badarg ->
350 648 ?LOG_INFO(#{what => mongoose_instrument_lookup_without_persistent_term,
351 648 event_name => EventName, labels => Labels}),
352 648 gen_server:call(?MODULE, {lookup, EventName, Labels})
353 end.
354
355 -spec lookup(event_name(), labels(), event_map()) -> {ok, handlers()} | {error, map()}.
356 lookup(EventName, Labels, Events) ->
357 378343 case Events of
358 #{EventName := #{Labels := Handlers}} ->
359 378343 {ok, Handlers};
360 #{} ->
361 
:-(
 {error, #{what => event_not_registered, event_name => EventName, labels => Labels}}
362 end.
363
364 -spec label_keys(labels()) -> [atom()].
365 label_keys(Labels) ->
366 44086 lists:sort(maps:keys(Labels)).
367
368 -spec get_handlers(event_name(), labels()) -> handlers().
369 get_handlers(EventName, Labels) ->
370 378343 case lookup(EventName, Labels) of
371 378343 {ok, Handlers} -> Handlers;
372 
:-(
 {error, ErrorMap} -> error(ErrorMap)
373 end.
374
375 -spec handle_event(event_name(), labels(), measurements(), handlers()) -> ok.
376 handle_event(EventName, Labels, Measurements, {EventHandlers, Config}) ->
377 378343 lists:foreach(fun(HandlerFun) ->
378 1153954 call_handler(HandlerFun, EventName, Labels, Config, Measurements)
379 end, EventHandlers).
380
381 -spec modules_to_funs([module()]) -> [handler_fun()].
382 modules_to_funs(Modules) ->
383 34143 [fun Module:handle_event/4 || Module <- Modules].
384
385 -spec handler_modules() -> [module()].
386 handler_modules() ->
387 26163 Keys = [Key || {Key, #{}} <- maps:to_list(mongoose_config:get_opt(instrumentation))],
388 26163 lists:map(fun handler_module/1, Keys).
389
390 -spec handler_module(handler_key()) -> module().
391 handler_module(Key) ->
392 80832 list_to_existing_atom("mongoose_instrument_" ++ atom_to_list(Key)).
393
394 -spec config_spec(handler_key()) -> mongoose_config_spec:config_section().
395 config_spec(Key) ->
396 279 Module = handler_module(Key),
397 279 case mongoose_lib:is_exported(Module, config_spec, 0) of
398 186 true -> Module:config_spec();
399 93 false -> #section{}
400 end.
401
402 -spec all_handler_keys() -> [handler_key()].
403 all_handler_keys() ->
404 93 [prometheus, exometer, log].
405
406 -spec start_handler(module()) -> ok.
407 start_handler(Module) ->
408 286 case mongoose_lib:is_exported(Module, start, 0) of
409 100 true -> Module:start();
410 186 false -> ok
411 end.
412
413 -spec stop_handler(module()) -> ok.
414 stop_handler(Module) ->
415 286 case mongoose_lib:is_exported(Module, stop, 0) of
416 100 true -> Module:stop();
417 186 false -> ok
418 end.
419
420 -spec call_handler(handler_fun(), event_name(), labels(), config(), measurements()) -> any().
421 call_handler(HandlerFun, EventName, Labels, Config, Measurements) ->
422 1153954 safely:apply_and_log(HandlerFun, [EventName, Labels, Config, Measurements],
423 #{what => event_handler_failed, handler_fun => HandlerFun,
424 event_name => EventName, labels => Labels, config => Config,
425 measurements => Measurements}).
Line Hits Source