./ct_report/coverage/mongoose_instrument.COVER.html

1 -module(mongoose_instrument).
2
3 -behaviour(gen_server).
4
5 %% API
6 -export([config_spec/0,
7 start_link/0, persist/0,
8 set_up/1, set_up/3,
9 tear_down/1, tear_down/2,
10 span/4, span/5,
11 execute/3]).
12
13 %% Test API
14 -export([add_handler/2, remove_handler/1]).
15
16 %% gen_server callbacks
17 -export([init/1, handle_call/3, handle_cast/2, code_change/3, handle_info/2, terminate/2]).
18
19 -ignore_xref([start_link/0, set_up/3, tear_down/2, span/4, add_handler/2, remove_handler/1,
20 probe/3]).
21
22 -include("mongoose.hrl").
23 -include("mongoose_config_spec.hrl").
24
25 -type event_name() :: atom().
26 -type labels() :: #{host_type => mongooseim:host_type()}. % to be extended
27 -type label_key() :: host_type. % to be extended
28 -type label_value() :: mongooseim:host_type(). % to be extended
29 -type metrics() :: #{metric_name() => metric_type()}.
30 -type metric_name() :: atom().
31 -type metric_type() :: gauge | spiral | histogram. % to be extended
32 -type measurements() :: #{atom() => term()}.
33 -type spec() :: {event_name(), labels(), config()}.
34 -type config() :: #{metrics => metrics(),
35 loglevel => logger:level(),
36 probe => probe_config()}.
37 -type probe_config() :: #{module := module(),
38 interval => pos_integer}.
39 -type handler_key() :: atom(). % key in the `instrumentation' section of the config file
40 -type handler_fun() :: fun((event_name(), labels(), config(), measurements()) -> any()).
41 -type handlers() :: {[handler_fun()], config()}.
42 -type execution_time() :: integer().
43 -type measure_fun(Result) :: fun((execution_time(), Result) -> measurements()).
44
45 -callback config_spec() -> mongoose_config_spec:config_section().
46 -callback start() -> ok.
47 -callback stop() -> ok.
48 -callback set_up(event_name(), labels(), config()) -> boolean().
49 -callback handle_event(event_name(), labels(), config(), measurements()) -> any().
50
51 -optional_callbacks([config_spec/0, start/0, stop/0]).
52
53 -export_type([event_name/0, labels/0, label_key/0, label_value/0, config/0, measurements/0,
54 spec/0, handlers/0, metric_name/0, metric_type/0, probe_config/0]).
55
56 %% API
57
58 %% @doc Specifies the `instrumentation' section of the config file
59 -spec config_spec() -> mongoose_config_spec:config_section().
60 config_spec() ->
61 42 Items = [{atom_to_binary(Key), config_spec(Key)} || Key <- all_handler_keys()],
62 42 Options = #{<<"probe_interval">> => #option{type = integer, validate = positive}},
63 42 #section{items = maps:merge(maps:from_list(Items), Options),
64 defaults = #{<<"probe_interval">> => 15},
65 wrap = global_config,
66 include = always}.
67
68 -spec start_link() -> gen_server:start_ret().
69 start_link() ->
70 42 gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
71
72 %% @doc Saves the state to a persistent term, improving performance of `execute' and `span'.
73 %% On the other hand, future calls to `set_up' or `tear_down' will update the persistent term,
74 %% which makes them less performant.
75 %% You should call this function only once - after the initial setup, but before handling any load.
76 -spec persist() -> ok.
77 persist() ->
78 42 gen_server:call(?MODULE, persist).
79
80 %% @doc Sets up instrumentation for multiple events.
81 %% @see set_up/3
82 -spec set_up([spec()]) -> ok.
83 set_up(Specs) ->
84 285 lists:foreach(fun({EventName, Labels, Config}) -> set_up(EventName, Labels, Config) end, Specs).
85
86 %% @doc Tears down instrumentation for multiple events.
87 %% @see tear_down/2
88 -spec tear_down([spec()]) -> ok.
89 tear_down(Specs) ->
90 285 lists:foreach(fun({EventName, Labels, _Config}) -> tear_down(EventName, Labels) end, Specs).
91
92 %% @doc Sets up instrumentation for an event identified by `EventName' and `Labels'
93 %% according to `Config'. Fails if the event is already registered, or if the keys of `Labels'
94 %% are different than for already registered events with `EventName'.
95 -spec set_up(event_name(), labels(), config()) -> ok.
96 set_up(EventName, Labels, Config) ->
97 1607 case gen_server:call(?MODULE, {set_up, EventName, Labels, Config}) of
98 1607 ok -> ok;
99 
:-(
 {error, ErrorMap} -> error(ErrorMap)
100 end.
101
102 %% @doc Tears down instrumentation for an event identified by `EventName' and `Labels'.
103 %% This operation is idempotent.
104 -spec tear_down(event_name(), labels()) -> ok.
105 tear_down(EventName, Labels) ->
106 1607 gen_server:call(?MODULE, {tear_down, EventName, Labels}).
107
108 %% @doc Calls `F', measuring its result with `MeasureF', and calls attached event handlers.
109 %% @see span/5
110 -spec span(event_name(), labels(), fun(() -> Result), measure_fun(Result)) -> Result.
111 span(EventName, Labels, F, MeasureF) ->
112 
:-(
 span(EventName, Labels, F, [], MeasureF).
113
114 %% @doc Calls `F' with `Args', measuring its execution time.
115 %% The time and the result are passed to `MeasureF', which returns measurements.
116 %% The measurements are then passed to all handlers attached to
117 %% the event identified by `EventName' and `Labels'.
118 %% Fails without calling `F' if the event is not registered.
119 -spec span(event_name(), labels(), fun((...) -> Result), list(), measure_fun(Result)) -> Result.
120 span(EventName, Labels, F, Args, MeasureF) ->
121 13575 Handlers = get_handlers(EventName, Labels),
122 13575 {Time, Result} = timer:tc(F, Args),
123 13575 handle_event(EventName, Labels, MeasureF(Time, Result), Handlers),
124 13575 Result.
125
126 %% @doc Executes all handlers attached to the event identified by `EventName' and `Labels',
127 %% passing `Measurements' to them. Fails if the event is not registered.
128 -spec execute(event_name(), labels(), measurements()) -> ok.
129 execute(EventName, Labels, Measurements) ->
130 2003 Handlers = get_handlers(EventName, Labels),
131 2003 handle_event(EventName, Labels, Measurements, Handlers).
132
133 %% Test API
134
135 -spec add_handler(handler_key(), mongoose_config:value()) -> ok.
136 add_handler(Key, ConfigVal) ->
137 2 case gen_server:call(?MODULE, {add_handler, Key, ConfigVal}) of
138 2 ok -> ok;
139 
:-(
 {error, ErrorMap} -> error(ErrorMap)
140 end.
141
142 -spec remove_handler(handler_key()) -> ok.
143 remove_handler(Key) ->
144 2 case gen_server:call(?MODULE, {remove_handler, Key}) of
145 2 ok -> ok;
146 
:-(
 {error, ErrorMap} -> error(ErrorMap)
147 end.
148
149 %% gen_server callbacks
150
151 -type state() :: #{events := event_map(), probe_timers := probe_timer_map()}.
152 -type event_map() :: #{event_name() => #{labels() => handlers()}}.
153 -type probe_timer_map() :: #{{event_name(), labels()} => timer:tref()}.
154
155 -spec init([]) -> {ok, state()}.
156 init([]) ->
157 42 lists:foreach(fun start_handler/1, handler_modules()),
158 42 erlang:process_flag(trap_exit, true), % Make sure that terminate is called
159 42 persistent_term:erase(?MODULE), % Prevent inconsistency when restarted after a kill
160 42 {ok, #{events => #{}, probe_timers => #{}}}.
161
162 -spec handle_call(any(), gen_server:from(), state()) ->
163 {reply, ok | {ok, handlers()} | {error, map()}, state()}.
164 handle_call({set_up, EventName, Labels, Config}, _From,
165 #{events := Events, probe_timers := ProbeTimers} = State) ->
166 1607 case set_up_and_register_event(EventName, Labels, Config, Events) of
167 {error, _} = Error ->
168 
:-(
 {reply, Error, State};
169 NewEvents = #{} ->
170 1607 update_if_persisted(Events, NewEvents),
171 1607 NewProbeTimers = start_probe_if_needed(EventName, Labels, Config, ProbeTimers),
172 1607 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}}
173 end;
174 handle_call({tear_down, EventName, Labels}, _From,
175 #{events := Events, probe_timers := ProbeTimers}) ->
176 1607 NewProbeTimers = deregister_probe_timer(EventName, Labels, ProbeTimers),
177 1607 NewEvents = deregister_event(EventName, Labels, Events),
178 1607 update_if_persisted(Events, NewEvents),
179 1607 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}};
180 handle_call({add_handler, Key, ConfigOpts}, _From, State = #{events := Events}) ->
181 2 case mongoose_config:lookup_opt([instrumentation, Key]) of
182 {error, not_found} ->
183 2 mongoose_config:set_opt([instrumentation, Key], ConfigOpts),
184 2 Module = handler_module(Key),
185 2 start_handler(Module),
186 2 NewEvents = update_handlers(Events, [], [Module]),
187 2 update_if_persisted(Events, NewEvents),
188 2 {reply, ok, State#{events := NewEvents}};
189 {ok, ExistingConfig} ->
190 
:-(
 {reply, {error, #{what => handler_already_configured, handler_key => Key,
191 existing_config => ExistingConfig}},
192 State}
193 end;
194 handle_call({remove_handler, Key}, _From, State = #{events := Events}) ->
195 2 case mongoose_config:lookup_opt([instrumentation, Key]) of
196 {error, not_found} ->
197 
:-(
 {reply, {error, #{what => handler_not_configured, handler_key => Key}}, State};
198 {ok, _} ->
199 2 mongoose_config:unset_opt([instrumentation, Key]),
200 2 Module = handler_module(Key),
201 2 NewEvents = update_handlers(Events, [Module], []),
202 2 update_if_persisted(Events, NewEvents),
203 2 stop_handler(Module),
204 2 {reply, ok, State#{events := NewEvents}}
205 end;
206 handle_call(persist, _From, State = #{events := Events}) ->
207 42 persistent_term:put(?MODULE, Events),
208 42 {reply, ok, State};
209 handle_call({lookup, EventName, Labels}, _From, State = #{events := Events}) ->
210 
:-(
 {reply, lookup(EventName, Labels, Events), State};
211 handle_call(Request, From, State) ->
212 
:-(
 ?UNEXPECTED_CALL(Request, From),
213 
:-(
 {reply, {error, #{what => unexpected_call, request => Request}}, State}.
214
215 -spec handle_cast(any(), state()) -> {noreply, state()}.
216 handle_cast(Msg, State) ->
217 
:-(
 ?UNEXPECTED_CAST(Msg),
218 
:-(
 {noreply, State}.
219
220 -spec handle_info(any(), state()) -> {noreply, state()}.
221 handle_info(Info, State) ->
222 
:-(
 ?UNEXPECTED_INFO(Info),
223 
:-(
 {noreply, State}.
224
225 -spec terminate(any(), state()) -> ok.
226 terminate(_Reason, _State) ->
227 42 persistent_term:erase(?MODULE),
228 42 lists:foreach(fun stop_handler/1, handler_modules()).
229
230 -spec code_change(any(), state(), any()) -> {ok, state()}.
231 code_change(_OldVsn, State, _Extra) ->
232 
:-(
 {ok, State}.
233
234 %% Internal functions
235
236 -spec update_if_persisted(event_map(), event_map()) -> ok.
237 update_if_persisted(Events, NewEvents) ->
238 3218 try persistent_term:get(?MODULE) of
239 3218 Events -> persistent_term:put(?MODULE, NewEvents)
240 catch
241 
:-(
 error:badarg -> ok
242 end.
243
244 -spec set_up_and_register_event(event_name(), labels(), config(), event_map()) ->
245 event_map() | {error, map()}.
246 set_up_and_register_event(EventName, Labels, Config, Events) ->
247 1607 LabelKeys = label_keys(Labels),
248 1607 case Events of
249 #{EventName := #{Labels := _}} ->
250 
:-(
 {error, #{what => event_already_registered,
251 event_name => EventName, labels => Labels}};
252 #{EventName := HandlerMap} ->
253 
:-(
 {ExistingLabels, _, _} = maps:next(maps:iterator(HandlerMap)),
254 
:-(
 case label_keys(ExistingLabels) of
255 LabelKeys ->
256 
:-(
 Handlers = do_set_up(EventName, Labels, Config),
257 
:-(
 Events#{EventName := HandlerMap#{Labels => Handlers}};
258 ExistingKeys ->
259 
:-(
 {error, #{what => inconsistent_labels,
260 event_name => EventName, labels => Labels,
261 existing_label_keys => ExistingKeys}}
262 end;
263 #{} ->
264 1607 Handlers = do_set_up(EventName, Labels, Config),
265 1607 Events#{EventName => #{Labels => Handlers}}
266 end.
267
268 -spec do_set_up(event_name(), labels(), config()) -> handlers().
269 do_set_up(EventName, Labels, Config) ->
270 1607 HandlerFuns = set_up_handlers(EventName, Labels, Config, handler_modules()),
271 1607 {HandlerFuns, Config}.
272
273 -spec start_probe_if_needed(event_name(), labels(), config(), probe_timer_map()) ->
274 probe_timer_map().
275 start_probe_if_needed(EventName, Labels, #{probe := _} = Config, ProbeTimers) ->
276 58 TRef = mongoose_instrument_probe:start_probe_timer(EventName, Labels, Config),
277 58 add_probe_timer(EventName, Labels, TRef, ProbeTimers);
278 start_probe_if_needed(_EventName, _Labels, _Config, ProbeTimers) ->
279 1549 ProbeTimers.
280
281 -spec add_probe_timer(event_name(), labels(), timer:tref(), probe_timer_map()) -> probe_timer_map().
282 add_probe_timer(EventName, Labels, TRef, ProbeTimers) ->
283 58 false = maps:is_key({EventName, Labels}, ProbeTimers), % sanity check to detect timer leak
284 58 ProbeTimers#{{EventName, Labels} => TRef}.
285
286 -spec update_handlers(event_map(), [module()], [module()]) -> event_map().
287 update_handlers(Events, ToRemove, ToAdd) ->
288 4 maps:map(fun(EventName, HandlerMap) ->
289 
:-(
 maps:map(fun(Labels, Handlers) ->
290 
:-(
 update_event_handlers(EventName, Labels, Handlers,
291 ToRemove, ToAdd)
292 end, HandlerMap)
293 end, Events).
294
295 -spec update_event_handlers(event_name(), labels(), handlers(), [module()], [module()]) ->
296 handlers().
297 update_event_handlers(EventName, Labels, {HandlerFuns, Config}, ToRemove, ToAdd) ->
298 
:-(
 FunsToRemove = modules_to_funs(ToRemove),
299 
:-(
 FunsToAdd = set_up_handlers(EventName, Labels, Config, ToAdd),
300 
:-(
 HandlerFuns = HandlerFuns -- FunsToAdd, % sanity check to prevent duplicates
301 
:-(
 {(HandlerFuns -- FunsToRemove) ++ FunsToAdd, Config}.
302
303 -spec set_up_handlers(event_name(), labels(), config(), [module()]) -> [handler_fun()].
304 set_up_handlers(EventName, Labels, Config, Modules) ->
305 1607 UsedModules = lists:filter(fun(Mod) -> Mod:set_up(EventName, Labels, Config) end, Modules),
306 1607 modules_to_funs(UsedModules).
307
308 -spec deregister_event(event_name(), labels(), event_map()) -> event_map().
309 deregister_event(EventName, Labels, Events) ->
310 1607 case Events of
311 #{EventName := HandlerMap} ->
312 1607 case maps:remove(Labels, HandlerMap) of
313 Empty when Empty =:= #{} ->
314 1607 maps:remove(EventName, Events);
315 NewHandlerMap ->
316 
:-(
 Events#{EventName := NewHandlerMap}
317 end;
318 #{} ->
319 
:-(
 Events
320 end.
321
322 -spec deregister_probe_timer(event_name(), labels(), probe_timer_map()) -> probe_timer_map().
323 deregister_probe_timer(EventName, Labels, ProbeTimers) ->
324 1607 case maps:take({EventName, Labels}, ProbeTimers) of
325 {TRef, NewProbeTimers} ->
326 58 timer:cancel(TRef),
327 58 NewProbeTimers;
328 error ->
329 1549 ProbeTimers % no timer was registered
330 end.
331
332 -spec lookup(event_name(), labels()) -> {ok, handlers()} | {error, map()}.
333 lookup(EventName, Labels) ->
334 15578 try persistent_term:get(?MODULE) of
335 Events ->
336 15578 lookup(EventName, Labels, Events)
337 catch
338 %% Although persist/0 should be called before handling traffic,
339 %% some instrumented events might happen before that, and they shouldn't fail.
340 error:badarg ->
341 
:-(
 ?LOG_INFO(#{what => mongoose_instrument_lookup_without_persistent_term,
342 
:-(
 event_name => EventName, labels => Labels}),
343 
:-(
 gen_server:call(?MODULE, {lookup, EventName, Labels})
344 end.
345
346 -spec lookup(event_name(), labels(), event_map()) -> {ok, handlers()} | {error, map()}.
347 lookup(EventName, Labels, Events) ->
348 15578 case Events of
349 #{EventName := #{Labels := Handlers}} ->
350 15578 {ok, Handlers};
351 #{} ->
352 
:-(
 {error, #{what => event_not_registered, event_name => EventName, labels => Labels}}
353 end.
354
355 -spec label_keys(labels()) -> [atom()].
356 label_keys(Labels) ->
357 1607 lists:sort(maps:keys(Labels)).
358
359 -spec get_handlers(event_name(), labels()) -> handlers().
360 get_handlers(EventName, Labels) ->
361 15578 case lookup(EventName, Labels) of
362 15578 {ok, Handlers} -> Handlers;
363 
:-(
 {error, ErrorMap} -> error(ErrorMap)
364 end.
365
366 -spec handle_event(event_name(), labels(), measurements(), handlers()) -> ok.
367 handle_event(EventName, Labels, Measurements, {EventHandlers, Config}) ->
368 15578 lists:foreach(fun(HandlerFun) ->
369 61293 call_handler(HandlerFun, EventName, Labels, Config, Measurements)
370 end, EventHandlers).
371
372 -spec modules_to_funs([module()]) -> [handler_fun()].
373 modules_to_funs(Modules) ->
374 1607 [fun Module:handle_event/4 || Module <- Modules].
375
376 -spec handler_modules() -> [module()].
377 handler_modules() ->
378 1691 Keys = [Key || {Key, #{}} <- maps:to_list(mongoose_config:get_opt(instrumentation))],
379 1649 lists:map(fun handler_module/1, Keys).
380
381 -spec handler_module(handler_key()) -> module().
382 handler_module(Key) ->
383 6231 list_to_existing_atom("mongoose_instrument_" ++ atom_to_list(Key)).
384
385 -spec config_spec(handler_key()) -> mongoose_config_spec:config_section().
386 config_spec(Key) ->
387 126 Module = handler_module(Key),
388 126 case mongoose_lib:is_exported(Module, config_spec, 0) of
389 84 true -> Module:config_spec();
390 42 false -> #section{}
391 end.
392
393 -spec all_handler_keys() -> [handler_key()].
394 all_handler_keys() ->
395 42 [prometheus, exometer, log].
396
397 -spec start_handler(module()) -> ok.
398 start_handler(Module) ->
399 128 case mongoose_lib:is_exported(Module, start, 0) of
400 44 true -> Module:start();
401 84 false -> ok
402 end.
403
404 -spec stop_handler(module()) -> ok.
405 stop_handler(Module) ->
406 2 case mongoose_lib:is_exported(Module, stop, 0) of
407 2 true -> Module:stop();
408 
:-(
 false -> ok
409 end.
410
411 -spec call_handler(handler_fun(), event_name(), labels(), config(), measurements()) -> any().
412 call_handler(HandlerFun, EventName, Labels, Config, Measurements) ->
413 61293 safely:apply_and_log(HandlerFun, [EventName, Labels, Config, Measurements],
414 #{what => event_handler_failed, handler_fun => HandlerFun,
415 event_name => EventName, labels => Labels, config => Config,
416 measurements => Measurements}).
Line Hits Source