./ct_report/coverage/mongoose_instrument.COVER.html

1 -module(mongoose_instrument).
2
3 -behaviour(gen_server).
4
5 %% API
6 -export([config_spec/0,
7 start_link/0, persist/0,
8 set_up/1, set_up/3,
9 tear_down/1, tear_down/2,
10 span/4, span/5,
11 execute/3]).
12
13 %% Test API
14 -export([add_handler/2, remove_handler/1]).
15
16 %% gen_server callbacks
17 -export([init/1, handle_call/3, handle_cast/2, code_change/3, handle_info/2, terminate/2]).
18
19 -ignore_xref([start_link/0, set_up/3, tear_down/2, span/4, add_handler/2, remove_handler/1]).
20
21 -include("mongoose.hrl").
22 -include("mongoose_config_spec.hrl").
23
24 -type event_name() :: atom().
25 -type labels() :: #{host_type => mongooseim:host_type()}. % to be extended
26 -type label_key() :: host_type. % to be extended
27 -type label_value() :: mongooseim:host_type(). % to be extended
28 -type metrics() :: #{metric_name() => metric_type()}.
29 -type metric_name() :: atom().
30 -type metric_type() :: gauge | spiral | histogram. % to be extended
31 -type measurements() :: #{atom() => term()}.
32 -type spec() :: {event_name(), labels(), config()}.
33 -type config() :: #{metrics => metrics(),
34 loglevel => logger:level(),
35 probe => probe_config()}.
36 -type probe_config() :: #{module := module(),
37 interval => pos_integer()}.
38 -type handler_key() :: atom(). % key in the `instrumentation' section of the config file
39 -type handler_fun() :: fun((event_name(), labels(), config(), measurements()) -> any()).
40 -type handlers() :: {[handler_fun()], config()}.
41 -type execution_time() :: integer().
42 -type measure_fun(Result) :: fun((execution_time(), Result) -> measurements()).
43
44 -callback config_spec() -> mongoose_config_spec:config_section().
45 -callback start() -> ok.
46 -callback stop() -> ok.
47 -callback set_up(event_name(), labels(), config()) -> boolean().
48 -callback handle_event(event_name(), labels(), config(), measurements()) -> any().
49
50 -optional_callbacks([config_spec/0, start/0, stop/0]).
51
52 -export_type([event_name/0, labels/0, label_key/0, label_value/0, config/0, measurements/0,
53 spec/0, handlers/0, metric_name/0, metric_type/0, probe_config/0]).
54
55 %% API
56
57 %% @doc Specifies the `instrumentation' section of the config file
58 -spec config_spec() -> mongoose_config_spec:config_section().
59 config_spec() ->
60 103 Items = [{atom_to_binary(Key), config_spec(Key)} || Key <- all_handler_keys()],
61 103 Options = #{<<"probe_interval">> => #option{type = integer, validate = positive}},
62 103 #section{items = maps:merge(maps:from_list(Items), Options),
63 defaults = #{<<"probe_interval">> => 15},
64 wrap = global_config,
65 include = always}.
66
67 -spec start_link() -> gen_server:start_ret().
68 start_link() ->
69 103 gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
70
71 %% @doc Saves the state to a persistent term, improving performance of `execute' and `span'.
72 %% On the other hand, future calls to `set_up' or `tear_down' will update the persistent term,
73 %% which makes them less performant.
74 %% You should call this function only once - after the initial setup, but before handling any load.
75 -spec persist() -> ok.
76 persist() ->
77 103 gen_server:call(?MODULE, persist).
78
79 %% @doc Sets up instrumentation for multiple events.
80 %% @see set_up/3
81 -spec set_up([spec()]) -> ok.
82 set_up(Specs) ->
83 270 lists:foreach(fun({EventName, Labels, Config}) -> set_up(EventName, Labels, Config) end, Specs).
84
85 %% @doc Tears down instrumentation for multiple events.
86 %% @see tear_down/2
87 -spec tear_down([spec()]) -> ok.
88 tear_down(Specs) ->
89 270 lists:foreach(fun({EventName, Labels, _Config}) -> tear_down(EventName, Labels) end, Specs).
90
91 %% @doc Sets up instrumentation for an event identified by `EventName' and `Labels'
92 %% according to `Config'. Fails if the event is already registered, or if the keys of `Labels'
93 %% are different than for already registered events with `EventName'.
94 -spec set_up(event_name(), labels(), config()) -> ok.
95 set_up(EventName, Labels, Config) ->
96 1518 case gen_server:call(?MODULE, {set_up, EventName, Labels, Config}) of
97 1518 ok -> ok;
98 
:-(
 {error, ErrorMap} -> error(ErrorMap)
99 end.
100
101 %% @doc Tears down instrumentation for an event identified by `EventName' and `Labels'.
102 %% This operation is idempotent.
103 -spec tear_down(event_name(), labels()) -> ok.
104 tear_down(EventName, Labels) ->
105 1518 gen_server:call(?MODULE, {tear_down, EventName, Labels}).
106
107 %% @doc Calls `F', measuring its result with `MeasureF', and calls attached event handlers.
108 %% @see span/5
109 -spec span(event_name(), labels(), fun(() -> Result), measure_fun(Result)) -> Result.
110 span(EventName, Labels, F, MeasureF) ->
111 
:-(
 span(EventName, Labels, F, [], MeasureF).
112
113 %% @doc Calls `F' with `Args', measuring its execution time.
114 %% The time and the result are passed to `MeasureF', which returns measurements.
115 %% The measurements are then passed to all handlers attached to
116 %% the event identified by `EventName' and `Labels'.
117 %% Fails without calling `F' if the event is not registered.
118 -spec span(event_name(), labels(), fun((...) -> Result), list(), measure_fun(Result)) -> Result.
119 span(EventName, Labels, F, Args, MeasureF) ->
120 13733 Handlers = get_handlers(EventName, Labels),
121 13733 {Time, Result} = timer:tc(F, Args),
122 13733 handle_event(EventName, Labels, MeasureF(Time, Result), Handlers),
123 13733 Result.
124
125 %% @doc Executes all handlers attached to the event identified by `EventName' and `Labels',
126 %% passing `Measurements' to them. Fails if the event is not registered.
127 -spec execute(event_name(), labels(), measurements()) -> ok.
128 execute(EventName, Labels, Measurements) ->
129 1669 Handlers = get_handlers(EventName, Labels),
130 1669 handle_event(EventName, Labels, Measurements, Handlers).
131
132 %% Test API
133
134 -spec add_handler(handler_key(), mongoose_config:value()) -> ok.
135 add_handler(Key, ConfigVal) ->
136 1 case gen_server:call(?MODULE, {add_handler, Key, ConfigVal}) of
137 1 ok -> ok;
138 
:-(
 {error, ErrorMap} -> error(ErrorMap)
139 end.
140
141 -spec remove_handler(handler_key()) -> ok.
142 remove_handler(Key) ->
143 1 case gen_server:call(?MODULE, {remove_handler, Key}) of
144 1 ok -> ok;
145 
:-(
 {error, ErrorMap} -> error(ErrorMap)
146 end.
147
148 %% gen_server callbacks
149
150 -type state() :: #{events := event_map(), probe_timers := probe_timer_map()}.
151 -type event_map() :: #{event_name() => #{labels() => handlers()}}.
152 -type probe_timer_map() :: #{{event_name(), labels()} => timer:tref()}.
153
154 -spec init([]) -> {ok, state()}.
155 init([]) ->
156 103 lists:foreach(fun start_handler/1, handler_modules()),
157 103 erlang:process_flag(trap_exit, true), % Make sure that terminate is called
158 103 persistent_term:erase(?MODULE), % Prevent inconsistency when restarted after a kill
159 103 {ok, #{events => #{}, probe_timers => #{}}}.
160
161 -spec handle_call(any(), gen_server:from(), state()) ->
162 {reply, ok | {ok, handlers()} | {error, map()}, state()}.
163 handle_call({set_up, EventName, Labels, Config}, _From,
164 #{events := Events, probe_timers := ProbeTimers} = State) ->
165 1518 case set_up_and_register_event(EventName, Labels, Config, Events) of
166 {error, _} = Error ->
167 
:-(
 {reply, Error, State};
168 NewEvents = #{} ->
169 1518 update_if_persisted(Events, NewEvents),
170 1518 NewProbeTimers = start_probe_if_needed(EventName, Labels, Config, ProbeTimers),
171 1518 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}}
172 end;
173 handle_call({tear_down, EventName, Labels}, _From,
174 #{events := Events, probe_timers := ProbeTimers}) ->
175 1518 NewProbeTimers = deregister_probe_timer(EventName, Labels, ProbeTimers),
176 1518 NewEvents = deregister_event(EventName, Labels, Events),
177 1518 update_if_persisted(Events, NewEvents),
178 1518 {reply, ok, #{events => NewEvents, probe_timers => NewProbeTimers}};
179 handle_call({add_handler, Key, ConfigOpts}, _From, State = #{events := Events}) ->
180 1 case mongoose_config:lookup_opt([instrumentation, Key]) of
181 {error, not_found} ->
182 1 mongoose_config:set_opt([instrumentation, Key], ConfigOpts),
183 1 Module = handler_module(Key),
184 1 start_handler(Module),
185 1 NewEvents = update_handlers(Events, [], [Module]),
186 1 update_if_persisted(Events, NewEvents),
187 1 {reply, ok, State#{events := NewEvents}};
188 {ok, ExistingConfig} ->
189 
:-(
 {reply, {error, #{what => handler_already_configured, handler_key => Key,
190 existing_config => ExistingConfig}},
191 State}
192 end;
193 handle_call({remove_handler, Key}, _From, State = #{events := Events}) ->
194 1 case mongoose_config:lookup_opt([instrumentation, Key]) of
195 {error, not_found} ->
196 
:-(
 {reply, {error, #{what => handler_not_configured, handler_key => Key}}, State};
197 {ok, _} ->
198 1 mongoose_config:unset_opt([instrumentation, Key]),
199 1 Module = handler_module(Key),
200 1 NewEvents = update_handlers(Events, [Module], []),
201 1 update_if_persisted(Events, NewEvents),
202 1 stop_handler(Module),
203 1 {reply, ok, State#{events := NewEvents}}
204 end;
205 handle_call(persist, _From, State = #{events := Events}) ->
206 103 persistent_term:put(?MODULE, Events),
207 103 {reply, ok, State};
208 handle_call({lookup, EventName, Labels}, _From, State = #{events := Events}) ->
209 
:-(
 {reply, lookup(EventName, Labels, Events), State};
210 handle_call(Request, From, State) ->
211 
:-(
 ?UNEXPECTED_CALL(Request, From),
212 
:-(
 {reply, {error, #{what => unexpected_call, request => Request}}, State}.
213
214 -spec handle_cast(any(), state()) -> {noreply, state()}.
215 handle_cast(Msg, State) ->
216 
:-(
 ?UNEXPECTED_CAST(Msg),
217 
:-(
 {noreply, State}.
218
219 -spec handle_info(any(), state()) -> {noreply, state()}.
220 handle_info(Info, State) ->
221 
:-(
 ?UNEXPECTED_INFO(Info),
222 
:-(
 {noreply, State}.
223
224 -spec terminate(any(), state()) -> ok.
225 terminate(_Reason, _State) ->
226 104 persistent_term:erase(?MODULE),
227 104 lists:foreach(fun stop_handler/1, handler_modules()).
228
229 -spec code_change(any(), state(), any()) -> {ok, state()}.
230 code_change(_OldVsn, State, _Extra) ->
231 
:-(
 {ok, State}.
232
233 %% Internal functions
234
235 -spec update_if_persisted(event_map(), event_map()) -> ok.
236 update_if_persisted(Events, NewEvents) ->
237 3038 try persistent_term:get(?MODULE) of
238 3038 Events -> persistent_term:put(?MODULE, NewEvents)
239 catch
240 
:-(
 error:badarg -> ok
241 end.
242
243 -spec set_up_and_register_event(event_name(), labels(), config(), event_map()) ->
244 event_map() | {error, map()}.
245 set_up_and_register_event(EventName, Labels, Config, Events) ->
246 1518 LabelKeys = label_keys(Labels),
247 1518 case Events of
248 #{EventName := #{Labels := _}} ->
249 
:-(
 {error, #{what => event_already_registered,
250 event_name => EventName, labels => Labels}};
251 #{EventName := HandlerMap} ->
252 56 {ExistingLabels, _, _} = maps:next(maps:iterator(HandlerMap)),
253 56 case label_keys(ExistingLabels) of
254 LabelKeys ->
255 56 Handlers = do_set_up(EventName, Labels, Config),
256 56 Events#{EventName := HandlerMap#{Labels => Handlers}};
257 ExistingKeys ->
258 
:-(
 {error, #{what => inconsistent_labels,
259 event_name => EventName, labels => Labels,
260 existing_label_keys => ExistingKeys}}
261 end;
262 #{} ->
263 1462 Handlers = do_set_up(EventName, Labels, Config),
264 1462 Events#{EventName => #{Labels => Handlers}}
265 end.
266
267 -spec do_set_up(event_name(), labels(), config()) -> handlers().
268 do_set_up(EventName, Labels, Config) ->
269 1518 HandlerFuns = set_up_handlers(EventName, Labels, Config, handler_modules()),
270 1518 {HandlerFuns, Config}.
271
272 -spec start_probe_if_needed(event_name(), labels(), config(), probe_timer_map()) ->
273 probe_timer_map().
274 start_probe_if_needed(EventName, Labels, #{probe := ProbeConfig}, ProbeTimers) ->
275 
:-(
 TRef = mongoose_instrument_probe:start_probe_timer(EventName, Labels, ProbeConfig),
276 
:-(
 add_probe_timer(EventName, Labels, TRef, ProbeTimers);
277 start_probe_if_needed(_EventName, _Labels, _Config, ProbeTimers) ->
278 1518 ProbeTimers.
279
280 -spec add_probe_timer(event_name(), labels(), timer:tref(), probe_timer_map()) -> probe_timer_map().
281 add_probe_timer(EventName, Labels, TRef, ProbeTimers) ->
282 
:-(
 false = maps:is_key({EventName, Labels}, ProbeTimers), % sanity check to detect timer leak
283 
:-(
 ProbeTimers#{{EventName, Labels} => TRef}.
284
285 -spec update_handlers(event_map(), [module()], [module()]) -> event_map().
286 update_handlers(Events, ToRemove, ToAdd) ->
287 2 maps:map(fun(EventName, HandlerMap) ->
288 
:-(
 maps:map(fun(Labels, Handlers) ->
289 
:-(
 update_event_handlers(EventName, Labels, Handlers,
290 ToRemove, ToAdd)
291 end, HandlerMap)
292 end, Events).
293
294 -spec update_event_handlers(event_name(), labels(), handlers(), [module()], [module()]) ->
295 handlers().
296 update_event_handlers(EventName, Labels, {HandlerFuns, Config}, ToRemove, ToAdd) ->
297 
:-(
 FunsToRemove = modules_to_funs(ToRemove),
298 
:-(
 FunsToAdd = set_up_handlers(EventName, Labels, Config, ToAdd),
299 
:-(
 HandlerFuns = HandlerFuns -- FunsToAdd, % sanity check to prevent duplicates
300 
:-(
 {(HandlerFuns -- FunsToRemove) ++ FunsToAdd, Config}.
301
302 -spec set_up_handlers(event_name(), labels(), config(), [module()]) -> [handler_fun()].
303 set_up_handlers(EventName, Labels, Config, Modules) ->
304 1518 UsedModules = lists:filter(fun(Mod) -> Mod:set_up(EventName, Labels, Config) end, Modules),
305 1518 modules_to_funs(UsedModules).
306
307 -spec deregister_event(event_name(), labels(), event_map()) -> event_map().
308 deregister_event(EventName, Labels, Events) ->
309 1518 case Events of
310 #{EventName := HandlerMap} ->
311 1518 case maps:remove(Labels, HandlerMap) of
312 Empty when Empty =:= #{} ->
313 1462 maps:remove(EventName, Events);
314 NewHandlerMap ->
315 56 Events#{EventName := NewHandlerMap}
316 end;
317 #{} ->
318 
:-(
 Events
319 end.
320
321 -spec deregister_probe_timer(event_name(), labels(), probe_timer_map()) -> probe_timer_map().
322 deregister_probe_timer(EventName, Labels, ProbeTimers) ->
323 1518 case maps:take({EventName, Labels}, ProbeTimers) of
324 {TRef, NewProbeTimers} ->
325 
:-(
 timer:cancel(TRef),
326 
:-(
 NewProbeTimers;
327 error ->
328 1518 ProbeTimers % no timer was registered
329 end.
330
331 -spec lookup(event_name(), labels()) -> {ok, handlers()} | {error, map()}.
332 lookup(EventName, Labels) ->
333 15402 try persistent_term:get(?MODULE) of
334 Events ->
335 15402 lookup(EventName, Labels, Events)
336 catch
337 %% Although persist/0 should be called before handling traffic,
338 %% some instrumented events might happen before that, and they shouldn't fail.
339 error:badarg ->
340 
:-(
 ?LOG_INFO(#{what => mongoose_instrument_lookup_without_persistent_term,
341 
:-(
 event_name => EventName, labels => Labels}),
342 
:-(
 gen_server:call(?MODULE, {lookup, EventName, Labels})
343 end.
344
345 -spec lookup(event_name(), labels(), event_map()) -> {ok, handlers()} | {error, map()}.
346 lookup(EventName, Labels, Events) ->
347 15402 case Events of
348 #{EventName := #{Labels := Handlers}} ->
349 15402 {ok, Handlers};
350 #{} ->
351 
:-(
 {error, #{what => event_not_registered, event_name => EventName, labels => Labels}}
352 end.
353
354 -spec label_keys(labels()) -> [atom()].
355 label_keys(Labels) ->
356 1574 lists:sort(maps:keys(Labels)).
357
358 -spec get_handlers(event_name(), labels()) -> handlers().
359 get_handlers(EventName, Labels) ->
360 15402 case lookup(EventName, Labels) of
361 15402 {ok, Handlers} -> Handlers;
362 
:-(
 {error, ErrorMap} -> error(ErrorMap)
363 end.
364
365 -spec handle_event(event_name(), labels(), measurements(), handlers()) -> ok.
366 handle_event(EventName, Labels, Measurements, {EventHandlers, Config}) ->
367 15402 lists:foreach(fun(HandlerFun) ->
368 60633 call_handler(HandlerFun, EventName, Labels, Config, Measurements)
369 end, EventHandlers).
370
371 -spec modules_to_funs([module()]) -> [handler_fun()].
372 modules_to_funs(Modules) ->
373 1518 [fun Module:handle_event/4 || Module <- Modules].
374
375 -spec handler_modules() -> [module()].
376 handler_modules() ->
377 1725 Keys = [Key || {Key, #{}} <- maps:to_list(mongoose_config:get_opt(instrumentation))],
378 1622 lists:map(fun handler_module/1, Keys).
379
380 -spec handler_module(handler_key()) -> module().
381 handler_module(Key) ->
382 6275 list_to_existing_atom("mongoose_instrument_" ++ atom_to_list(Key)).
383
384 -spec config_spec(handler_key()) -> mongoose_config_spec:config_section().
385 config_spec(Key) ->
386 309 Module = handler_module(Key),
387 309 case mongoose_lib:is_exported(Module, config_spec, 0) of
388 206 true -> Module:config_spec();
389 103 false -> #section{}
390 end.
391
392 -spec all_handler_keys() -> [handler_key()].
393 all_handler_keys() ->
394 103 [prometheus, exometer, log].
395
396 -spec start_handler(module()) -> ok.
397 start_handler(Module) ->
398 310 case mongoose_lib:is_exported(Module, start, 0) of
399 104 true -> Module:start();
400 206 false -> ok
401 end.
402
403 -spec stop_handler(module()) -> ok.
404 stop_handler(Module) ->
405 4 case mongoose_lib:is_exported(Module, stop, 0) of
406 2 true -> Module:stop();
407 2 false -> ok
408 end.
409
410 -spec call_handler(handler_fun(), event_name(), labels(), config(), measurements()) -> any().
411 call_handler(HandlerFun, EventName, Labels, Config, Measurements) ->
412 60633 safely:apply_and_log(HandlerFun, [EventName, Labels, Config, Measurements],
413 #{what => event_handler_failed, handler_fun => HandlerFun,
414 event_name => EventName, labels => Labels, config => Config,
415 measurements => Measurements}).
Line Hits Source