./ct_report/coverage/mongoose_instrument.COVER.html

1 -module(mongoose_instrument).
2
3 -behaviour(gen_server).
4
5 %% API
6 -export([config_spec/0,
7 start_link/0, persist/0,
8 set_up/1, set_up/3,
9 tear_down/1, tear_down/2,
10 span/4, span/5,
11 execute/3]).
12
13 %% Test API
14 -export([add_handler/2, remove_handler/1]).
15
16 %% gen_server callbacks
17 -export([init/1, handle_call/3, handle_cast/2, code_change/3, handle_info/2, terminate/2]).
18
19 -ignore_xref([start_link/0, set_up/3, tear_down/2, span/4, add_handler/2, remove_handler/1,
20 probe/3]).
21
22 -include("mongoose.hrl").
23 -include("mongoose_config_spec.hrl").
24
25 -type event_name() :: atom().
26 -type labels() :: #{host_type => mongooseim:host_type()}. % to be extended
27 -type label_key() :: host_type. % to be extended
28 -type label_value() :: mongooseim:host_type(). % to be extended
29 -type metrics() :: #{metric_name() => metric_type()}.
30 -type metric_name() :: atom().
31 -type metric_type() :: gauge | spiral | histogram. % to be extended
32 -type measurements() :: #{atom() => term()}.
33 -type spec() :: {event_name(), labels(), config()}.
34 -type config() :: #{metrics => metrics(),
35 loglevel => logger:level(),
36 probe => module(),
37 probe_interval => pos_integer()}. % to be extended
38 -type handler_key() :: atom(). % key in the `instrumentation' section of the config file
39 -type handler_fun() :: fun((event_name(), labels(), config(), measurements()) -> any()).
40 -type handlers() :: {[handler_fun()], config()}.
41 -type execution_time() :: integer().
42 -type measure_fun(Result) :: fun((execution_time(), Result) -> measurements()).
43
44 -callback config_spec() -> mongoose_config_spec:config_section().
45 -callback start() -> ok.
46 -callback stop() -> ok.
47 -callback set_up(event_name(), labels(), config()) -> boolean().
48 -callback handle_event(event_name(), labels(), config(), measurements()) -> any().
49
50 -optional_callbacks([config_spec/0, start/0, stop/0]).
51
52 -export_type([event_name/0, labels/0, label_key/0, label_value/0, config/0, measurements/0,
53 spec/0, handlers/0, metric_name/0, metric_type/0]).
54
55 %% API
56
57 %% @doc Specifies the `instrumentation' section of the config file
58 -spec config_spec() -> mongoose_config_spec:config_section().
59 config_spec() ->
60 33 Items = [{atom_to_binary(Key), config_spec(Key)} || Key <- all_handler_keys()],
61 33 #section{items = maps:from_list(Items),
62 wrap = global_config,
63 include = always}.
64
65 -spec start_link() -> gen_server:start_ret().
66 start_link() ->
67 33 gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
68
69 %% @doc Saves the state to a persistent term, improving performance of `execute' and `span'.
70 %% On the other hand, future calls to `set_up' or `tear_down' will update the persistent term,
71 %% which makes them less performant.
72 %% You should call this function only once - after the initial setup, but before handling any load.
73 -spec persist() -> ok.
74 persist() ->
75 33 gen_server:call(?MODULE, persist).
76
77 %% @doc Sets up instrumentation for multiple events.
78 %% @see set_up/3
79 -spec set_up([spec()]) -> ok.
80 set_up(Specs) ->
81 264 lists:foreach(fun({EventName, Labels, Config}) -> set_up(EventName, Labels, Config) end, Specs).
82
83 %% @doc Tears down instrumentation for multiple events.
84 %% @see tear_down/2
85 -spec tear_down([spec()]) -> ok.
86 tear_down(Specs) ->
87 264 lists:foreach(fun({EventName, Labels, _Config}) -> tear_down(EventName, Labels) end, Specs).
88
89 %% @doc Sets up instrumentation for an event identified by `EventName' and `Labels'
90 %% according to `Config'. Fails if the event is already registered, or if the keys of `Labels'
91 %% are different than for already registered events with `EventName'.
92 -spec set_up(event_name(), labels(), config()) -> ok.
93 set_up(EventName, Labels, Config) ->
94 1400 case gen_server:call(?MODULE, {set_up, EventName, Labels, Config}) of
95 1400 ok -> ok;
96 
:-(
 {error, ErrorMap} -> error(ErrorMap)
97 end.
98
99 %% @doc Tears down instrumentation for an event identified by `EventName' and `Labels'.
100 %% This operation is idempotent.
101 -spec tear_down(event_name(), labels()) -> ok.
102 tear_down(EventName, Labels) ->
103 1400 gen_server:call(?MODULE, {tear_down, EventName, Labels}).
104
105 %% @doc Calls `F', measuring its result with `MeasureF', and calls attached event handlers.
106 %% @see span/5
107 -spec span(event_name(), labels(), fun(() -> Result), measure_fun(Result)) -> Result.
108 span(EventName, Labels, F, MeasureF) ->
109 
:-(
 span(EventName, Labels, F, [], MeasureF).
110
111 %% @doc Calls `F' with `Args', measuring its execution time.
112 %% The time and the result are passed to `MeasureF', which returns measurements.
113 %% The measurements are then passed to all handlers attached to
114 %% the event identified by `EventName' and `Labels'.
115 %% Fails without calling `F' if the event is not registered.
116 -spec span(event_name(), labels(), fun((...) -> Result), list(), measure_fun(Result)) -> Result.
117 span(EventName, Labels, F, Args, MeasureF) ->
118 10316 Handlers = get_handlers(EventName, Labels),
119 10316 {Time, Result} = timer:tc(F, Args),
120 10316 handle_event(EventName, Labels, MeasureF(Time, Result), Handlers),
121 10316 Result.
122
123 %% @doc Executes all handlers attached to the event identified by `EventName' and `Labels',
124 %% passing `Measurements' to them. Fails if the event is not registered.
125 -spec execute(event_name(), labels(), measurements()) -> ok.
126 execute(EventName, Labels, Measurements) ->
127 1479 Handlers = get_handlers(EventName, Labels),
128 1479 handle_event(EventName, Labels, Measurements, Handlers).
129
130 %% Test API
131
132 -spec add_handler(handler_key(), mongoose_config:value()) -> ok.
133 add_handler(Key, ConfigVal) ->
134 1 case gen_server:call(?MODULE, {add_handler, Key, ConfigVal}) of
135 1 ok -> ok;
136 
:-(
 {error, ErrorMap} -> error(ErrorMap)
137 end.
138
139 -spec remove_handler(handler_key()) -> ok.
140 remove_handler(Key) ->
141 1 case gen_server:call(?MODULE, {remove_handler, Key}) of
142 1 ok -> ok;
143 
:-(
 {error, ErrorMap} -> error(ErrorMap)
144 end.
145
146 %% gen_server callbacks
147
148 -type state() :: #{events := event_map(), probe_timers := probe_timer_map()}.
149 -type event_map() :: #{event_name() => #{labels() => handlers()}}.
150 -type probe_timer_map() :: #{{event_name(), labels()} => timer:tref()}.
151
152 -spec init([]) -> {ok, state()}.
153 init([]) ->
154 33 lists:foreach(fun start_handler/1, handler_modules()),
155 33 erlang:process_flag(trap_exit, true), % Make sure that terminate is called
156 33 persistent_term:erase(?MODULE), % Prevent inconsistency when restarted after a kill
157 33 {ok, #{events => #{}, probe_timers => #{}}}.
158
159 -spec handle_call(any(), gen_server:from(), state()) ->
160 {reply, ok | {ok, handlers()} | {error, map()}, state()}.
161 handle_call({set_up, EventName, Labels, Config}, _From,
162 #{events := Events, probe_timers := ProbeTimers} = State) ->
163 1400 case set_up_and_register_event(EventName, Labels, Config, Events) of
164 {error, _} = Error ->
165 
:-(
 {reply, Error, State};
166 NewEvents = #{} ->
167 1400 update_if_persisted(Events, NewEvents),
168 1400 NewProbeTimers = start_probe_if_needed(EventName, Labels, Config, ProbeTimers),
169 1400 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}}
170 end;
171 handle_call({tear_down, EventName, Labels}, _From,
172 #{events := Events, probe_timers := ProbeTimers}) ->
173 1400 NewProbeTimers = deregister_probe_timer(EventName, Labels, ProbeTimers),
174 1400 NewEvents = deregister_event(EventName, Labels, Events),
175 1400 update_if_persisted(Events, NewEvents),
176 1400 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}};
177 handle_call({add_handler, Key, ConfigOpts}, _From, State = #{events := Events}) ->
178 1 case mongoose_config:lookup_opt([instrumentation, Key]) of
179 {error, not_found} ->
180 1 mongoose_config:set_opt([instrumentation, Key], ConfigOpts),
181 1 Module = handler_module(Key),
182 1 start_handler(Module),
183 1 NewEvents = update_handlers(Events, [], [Module]),
184 1 update_if_persisted(Events, NewEvents),
185 1 {reply, ok, State#{events := NewEvents}};
186 {ok, ExistingConfig} ->
187 
:-(
 {reply, {error, #{what => handler_already_configured, handler_key => Key,
188 existing_config => ExistingConfig}},
189 State}
190 end;
191 handle_call({remove_handler, Key}, _From, State = #{events := Events}) ->
192 1 case mongoose_config:lookup_opt([instrumentation, Key]) of
193 {error, not_found} ->
194 
:-(
 {reply, {error, #{what => handler_not_configured, handler_key => Key}}, State};
195 {ok, _} ->
196 1 mongoose_config:unset_opt([instrumentation, Key]),
197 1 Module = handler_module(Key),
198 1 NewEvents = update_handlers(Events, [Module], []),
199 1 update_if_persisted(Events, NewEvents),
200 1 stop_handler(Module),
201 1 {reply, ok, State#{events := NewEvents}}
202 end;
203 handle_call(persist, _From, State = #{events := Events}) ->
204 33 persistent_term:put(?MODULE, Events),
205 33 {reply, ok, State};
206 handle_call({lookup, EventName, Labels}, _From, State = #{events := Events}) ->
207 
:-(
 {reply, lookup(EventName, Labels, Events), State};
208 handle_call(Request, From, State) ->
209 
:-(
 ?UNEXPECTED_CALL(Request, From),
210 
:-(
 {reply, {error, #{what => unexpected_call, request => Request}}, State}.
211
212 -spec handle_cast(any(), state()) -> {noreply, state()}.
213 handle_cast(Msg, State) ->
214 
:-(
 ?UNEXPECTED_CAST(Msg),
215 
:-(
 {noreply, State}.
216
217 -spec handle_info(any(), state()) -> {noreply, state()}.
218 handle_info(Info, State) ->
219 
:-(
 ?UNEXPECTED_INFO(Info),
220 
:-(
 {noreply, State}.
221
222 -spec terminate(any(), state()) -> ok.
223 terminate(_Reason, _State) ->
224 34 persistent_term:erase(?MODULE),
225 34 lists:foreach(fun stop_handler/1, handler_modules()).
226
227 -spec code_change(any(), state(), any()) -> {ok, state()}.
228 code_change(_OldVsn, State, _Extra) ->
229 
:-(
 {ok, State}.
230
231 %% Internal functions
232
233 -spec update_if_persisted(event_map(), event_map()) -> ok.
234 update_if_persisted(Events, NewEvents) ->
235 2802 try persistent_term:get(?MODULE) of
236 2802 Events -> persistent_term:put(?MODULE, NewEvents)
237 catch
238 
:-(
 error:badarg -> ok
239 end.
240
241 -spec set_up_and_register_event(event_name(), labels(), config(), event_map()) ->
242 event_map() | {error, map()}.
243 set_up_and_register_event(EventName, Labels, Config, Events) ->
244 1400 LabelKeys = label_keys(Labels),
245 1400 case Events of
246 #{EventName := #{Labels := _}} ->
247 
:-(
 {error, #{what => event_already_registered,
248 event_name => EventName, labels => Labels}};
249 #{EventName := HandlerMap} ->
250 86 {ExistingLabels, _, _} = maps:next(maps:iterator(HandlerMap)),
251 86 case label_keys(ExistingLabels) of
252 LabelKeys ->
253 86 Handlers = do_set_up(EventName, Labels, Config),
254 86 Events#{EventName := HandlerMap#{Labels => Handlers}};
255 ExistingKeys ->
256 
:-(
 {error, #{what => inconsistent_labels,
257 event_name => EventName, labels => Labels,
258 existing_label_keys => ExistingKeys}}
259 end;
260 #{} ->
261 1314 Handlers = do_set_up(EventName, Labels, Config),
262 1314 Events#{EventName => #{Labels => Handlers}}
263 end.
264
265 -spec do_set_up(event_name(), labels(), config()) -> handlers().
266 do_set_up(EventName, Labels, Config) ->
267 1400 HandlerFuns = set_up_handlers(EventName, Labels, Config, handler_modules()),
268 1400 {HandlerFuns, Config}.
269
270 -spec start_probe_if_needed(event_name(), labels(), config(), probe_timer_map()) ->
271 probe_timer_map().
272 start_probe_if_needed(EventName, Labels, #{probe := _} = Config, ProbeTimers) ->
273 76 TRef = mongoose_instrument_probe:start_probe_timer(EventName, Labels, Config),
274 76 add_probe_timer(EventName, Labels, TRef, ProbeTimers);
275 start_probe_if_needed(_EventName, _Labels, _Config, ProbeTimers) ->
276 1324 ProbeTimers.
277
278 -spec add_probe_timer(event_name(), labels(), timer:tref(), probe_timer_map()) -> probe_timer_map().
279 add_probe_timer(EventName, Labels, TRef, ProbeTimers) ->
280 76 false = maps:is_key({EventName, Labels}, ProbeTimers), % sanity check to detect timer leak
281 76 ProbeTimers#{{EventName, Labels} => TRef}.
282
283 -spec update_handlers(event_map(), [module()], [module()]) -> event_map().
284 update_handlers(Events, ToRemove, ToAdd) ->
285 2 maps:map(fun(EventName, HandlerMap) ->
286 
:-(
 maps:map(fun(Labels, Handlers) ->
287 
:-(
 update_event_handlers(EventName, Labels, Handlers,
288 ToRemove, ToAdd)
289 end, HandlerMap)
290 end, Events).
291
292 -spec update_event_handlers(event_name(), labels(), handlers(), [module()], [module()]) ->
293 handlers().
294 update_event_handlers(EventName, Labels, {HandlerFuns, Config}, ToRemove, ToAdd) ->
295 
:-(
 FunsToRemove = modules_to_funs(ToRemove),
296 
:-(
 FunsToAdd = set_up_handlers(EventName, Labels, Config, ToAdd),
297 
:-(
 HandlerFuns = HandlerFuns -- FunsToAdd, % sanity check to prevent duplicates
298 
:-(
 {(HandlerFuns -- FunsToRemove) ++ FunsToAdd, Config}.
299
300 -spec set_up_handlers(event_name(), labels(), config(), [module()]) -> [handler_fun()].
301 set_up_handlers(EventName, Labels, Config, Modules) ->
302 1400 UsedModules = lists:filter(fun(Mod) -> Mod:set_up(EventName, Labels, Config) end, Modules),
303 1400 modules_to_funs(UsedModules).
304
305 -spec deregister_event(event_name(), labels(), event_map()) -> event_map().
306 deregister_event(EventName, Labels, Events) ->
307 1400 case Events of
308 #{EventName := HandlerMap} ->
309 1400 case maps:remove(Labels, HandlerMap) of
310 Empty when Empty =:= #{} ->
311 1314 maps:remove(EventName, Events);
312 NewHandlerMap ->
313 86 Events#{EventName := NewHandlerMap}
314 end;
315 #{} ->
316 
:-(
 Events
317 end.
318
319 -spec deregister_probe_timer(event_name(), labels(), probe_timer_map()) -> probe_timer_map().
320 deregister_probe_timer(EventName, Labels, ProbeTimers) ->
321 1400 case maps:take({EventName, Labels}, ProbeTimers) of
322 {TRef, NewProbeTimers} ->
323 76 timer:cancel(TRef),
324 76 NewProbeTimers;
325 error ->
326 1324 ProbeTimers % no timer was registered
327 end.
328
329 -spec lookup(event_name(), labels()) -> {ok, handlers()} | {error, map()}.
330 lookup(EventName, Labels) ->
331 11795 try persistent_term:get(?MODULE) of
332 Events ->
333 11795 lookup(EventName, Labels, Events)
334 catch
335 %% Although persist/0 should be called before handling traffic,
336 %% some instrumented events might happen before that, and they shouldn't fail.
337 error:badarg ->
338 
:-(
 ?LOG_INFO(#{what => mongoose_instrument_lookup_without_persistent_term,
339 
:-(
 event_name => EventName, labels => Labels}),
340 
:-(
 gen_server:call(?MODULE, {lookup, EventName, Labels})
341 end.
342
343 -spec lookup(event_name(), labels(), event_map()) -> {ok, handlers()} | {error, map()}.
344 lookup(EventName, Labels, Events) ->
345 11795 case Events of
346 #{EventName := #{Labels := Handlers}} ->
347 11795 {ok, Handlers};
348 #{} ->
349 
:-(
 {error, #{what => event_not_registered, event_name => EventName, labels => Labels}}
350 end.
351
352 -spec label_keys(labels()) -> [atom()].
353 label_keys(Labels) ->
354 1486 lists:sort(maps:keys(Labels)).
355
356 -spec get_handlers(event_name(), labels()) -> handlers().
357 get_handlers(EventName, Labels) ->
358 11795 case lookup(EventName, Labels) of
359 11795 {ok, Handlers} -> Handlers;
360 
:-(
 {error, ErrorMap} -> error(ErrorMap)
361 end.
362
363 -spec handle_event(event_name(), labels(), measurements(), handlers()) -> ok.
364 handle_event(EventName, Labels, Measurements, {EventHandlers, Config}) ->
365 11795 lists:foreach(fun(HandlerFun) ->
366 46009 call_handler(HandlerFun, EventName, Labels, Config, Measurements)
367 end, EventHandlers).
368
369 -spec modules_to_funs([module()]) -> [handler_fun()].
370 modules_to_funs(Modules) ->
371 1400 [fun Module:handle_event/4 || Module <- Modules].
372
373 -spec handler_modules() -> [module()].
374 handler_modules() ->
375 1467 [handler_module(Key) || Key <- maps:keys(mongoose_config:get_opt(instrumentation))].
376
377 -spec handler_module(handler_key()) -> module().
378 handler_module(Key) ->
379 5198 list_to_existing_atom("mongoose_instrument_" ++ atom_to_list(Key)).
380
381 -spec config_spec(handler_key()) -> mongoose_config_spec:config_section().
382 config_spec(Key) ->
383 99 Module = handler_module(Key),
384 99 case mongoose_lib:is_exported(Module, config_spec, 0) of
385 66 true -> Module:config_spec();
386 33 false -> #section{}
387 end.
388
389 -spec all_handler_keys() -> [handler_key()].
390 all_handler_keys() ->
391 33 [prometheus, exometer, log].
392
393 -spec start_handler(module()) -> ok.
394 start_handler(Module) ->
395 100 case mongoose_lib:is_exported(Module, start, 0) of
396 34 true -> Module:start();
397 66 false -> ok
398 end.
399
400 -spec stop_handler(module()) -> ok.
401 stop_handler(Module) ->
402 4 case mongoose_lib:is_exported(Module, stop, 0) of
403 2 true -> Module:stop();
404 2 false -> ok
405 end.
406
407 -spec call_handler(handler_fun(), event_name(), labels(), config(), measurements()) -> any().
408 call_handler(HandlerFun, EventName, Labels, Config, Measurements) ->
409 46009 safely:apply_and_log(HandlerFun, [EventName, Labels, Config, Measurements],
410 #{what => event_handler_failed, handler_fun => HandlerFun,
411 event_name => EventName, labels => Labels, config => Config,
412 measurements => Measurements}).
Line Hits Source