./ct_report/coverage/mongoose_metrics.COVER.html

1 %%==============================================================================
2 %% Copyright 2014 Erlang Solutions Ltd.
3 %%
4 %% Licensed under the Apache License, Version 2.0 (the "License");
5 %% you may not use this file except in compliance with the License.
6 %% You may obtain a copy of the License at
7 %%
8 %% http://www.apache.org/licenses/LICENSE-2.0
9 %%
10 %% Unless required by applicable law or agreed to in writing, software
11 %% distributed under the License is distributed on an "AS IS" BASIS,
12 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 %% See the License for the specific language governing permissions and
14 %% limitations under the License.
15 %%==============================================================================
16 -module(mongoose_metrics).
17
18 -include("mongoose.hrl").
19 -include("mongoose_metrics_definitions.hrl").
20
21 %% API
22 -export([init/0,
23 create_generic_hook_metric/2,
24 ensure_db_pool_metric/1,
25 update/3,
26 ensure_metric/3,
27 ensure_subscribed_metric/3,
28 get_metric_value/1,
29 get_metric_values/1,
30 get_metric_value/2,
31 sample_metric/1,
32 get_host_type_metric_names/1,
33 get_global_metric_names/0,
34 get_aggregated_values/1,
35 increment_generic_hook_metric/2,
36 get_rdbms_data_stats/0,
37 get_rdbms_data_stats/1,
38 get_dist_data_stats/0,
39 get_up_time/0,
40 get_mnesia_running_db_nodes_count/0,
41 remove_host_type_metrics/1,
42 remove_all_metrics/0,
43 get_report_interval/0
44 ]).
45
46 -ignore_xref([get_dist_data_stats/0, get_mnesia_running_db_nodes_count/0,
47 get_rdbms_data_stats/0, get_rdbms_data_stats/1, get_up_time/0,
48 remove_host_type_metrics/1, get_report_interval/0,
49 sample_metric/1]).
50
51 -define(PREFIXES, mongoose_metrics_prefixes).
52 -define(DEFAULT_REPORT_INTERVAL, 60000). %%60s
53
54 -type use_or_skip() :: use | skip.
55 -type hook_name() :: atom().
56 -type metric_name() :: atom() | list(atom() | binary()).
57 -type short_metric_type() :: spiral | histogram | counter | gauge.
58 -type metric_type() :: tuple() | short_metric_type().
59
60 %% ---------------------------------------------------------------------
61 %% API
62 %% ---------------------------------------------------------------------
63
64 -spec init() -> ok.
65 init() ->
66 83 prepare_prefixes(),
67 83 create_global_metrics(),
68 83 lists:foreach(
69 fun(HostType) ->
70 426 init_predefined_host_type_metrics(HostType)
71 end, ?ALL_HOST_TYPES),
72 83 init_subscriptions().
73
74 create_global_metrics() ->
75 83 lists:foreach(fun({Metric, FunSpec, DataPoints}) ->
76 166 FunSpecTuple = list_to_tuple(FunSpec ++ [DataPoints]),
77 166 catch ensure_metric(global, Metric, FunSpecTuple)
78 end, ?VM_STATS),
79 83 lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
80 ?GLOBAL_COUNTERS),
81 83 create_data_metrics().
82
83 -spec init_predefined_host_type_metrics(mongooseim:host_type()) -> ok.
84 init_predefined_host_type_metrics(HostType) ->
85 426 create_metrics(HostType),
86 426 Hooks = mongoose_metrics_hooks:get_hooks(HostType),
87 426 ejabberd_hooks:add(Hooks),
88 426 ok.
89
90 init_subscriptions() ->
91 83 Reporters = exometer_report:list_reporters(),
92 83 lists:foreach(
93 fun({Name, _ReporterPid}) ->
94 
:-(
 Interval = get_report_interval(),
95 
:-(
 subscribe_to_all(Name, Interval)
96 end, Reporters).
97
98 -spec create_generic_hook_metric(mongooseim:host_type(), atom()) ->
99 ok | {ok, already_present} | {error, any()}.
100 create_generic_hook_metric(HostType, Hook) ->
101 16445 UseOrSkip = filter_hook(Hook),
102 16445 do_create_generic_hook_metric(HostType, Hook, UseOrSkip).
103
104 % TODO: change to HostType after mongoose_wpool_rdbms
105 ensure_db_pool_metric({rdbms, Host, Tag} = Name) ->
106 
:-(
 ensure_metric(Host,
107 [data, rdbms, Tag],
108 {function, mongoose_metrics, get_rdbms_data_stats, [[Name]], proplist,
109 [workers | ?INET_STATS]}).
110
111 -spec update(HostType :: mongooseim:host_type_or_global(), Name :: term() | list(),
112 Change :: term()) -> any().
113 update(HostType, Name, Change) when is_list(Name) ->
114 375709 exometer:update(name_by_all_metrics_are_global(HostType, Name), Change);
115 update(HostType, Name, Change) ->
116 213538 update(HostType, [Name], Change).
117
118 -spec ensure_metric(mongooseim:host_type_or_global(), metric_name(), metric_type()) ->
119 ok | {ok, already_present} | {error, any()}.
120 ensure_metric(HostType, Metric, Type) when is_tuple(Type) ->
121 891 ensure_metric(HostType, Metric, Type, element(1, Type));
122 ensure_metric(HostType, Metric, Type) ->
123 56494 ensure_metric(HostType, Metric, Type, Type).
124
125 get_metric_value(HostType, Name) when is_list(Name) ->
126 165 get_metric_value(name_by_all_metrics_are_global(HostType, Name));
127 get_metric_value(HostType, Name) ->
128 162 get_metric_value(HostType, [Name]).
129
130 get_metric_value(Metric) ->
131 300 exometer:get_value(Metric).
132
133 get_metric_values(Metric) when is_list(Metric) ->
134 58 exometer:get_values(Metric);
135 get_metric_values(HostType) ->
136 176 exometer:get_values([HostType]).
137
138 %% Force update a probe metric
139 sample_metric(Metric) ->
140 6 exometer:sample(Metric).
141
142 get_host_type_metric_names(HostType) ->
143 87 HostTypeName = get_host_type_prefix(HostType),
144 87 [MetricName || {[_HostTypeName | MetricName], _, _} <- exometer:find_entries([HostTypeName])].
145
146 get_global_metric_names() ->
147 2 get_host_type_metric_names(global).
148
149 get_aggregated_values(Metric) ->
150 9130 exometer:aggregate([{{['_', Metric], '_', '_'}, [], [true]}], [one, count, value]).
151
152 -spec increment_generic_hook_metric(mongooseim:host_type(), atom()) -> ok | {error, any()}.
153 increment_generic_hook_metric(HostType, Hook) ->
154 185212 UseOrSkip = filter_hook(Hook),
155 185212 do_increment_generic_hook_metric(HostType, Hook, UseOrSkip).
156
157 get_rdbms_data_stats() ->
158 
:-(
 Pools = lists:filter(fun({Type, _Host, _Tag}) -> Type == rdbms end, mongoose_wpool:get_pools()),
159 
:-(
 get_rdbms_data_stats(Pools).
160
161 get_rdbms_data_stats(Pools) ->
162 
:-(
 RDBMSWorkers =
163 lists:flatmap(
164 fun({Type, Host, Tag}) ->
165 
:-(
 PoolName = mongoose_wpool:make_pool_name(Type, Host, Tag),
166 
:-(
 Wpool = wpool_pool:find_wpool(PoolName),
167 
:-(
 PoolSize = wpool_pool:wpool_get(size, Wpool),
168 
:-(
 [whereis(wpool_pool:worker_name(PoolName, I)) || I <- lists:seq(1, PoolSize)]
169 end,
170 Pools),
171
172 
:-(
 get_rdbms_stats(RDBMSWorkers).
173
174 get_dist_data_stats() ->
175 104 DistStats = [dist_inet_stats(PortOrPid) || {_, PortOrPid} <- erlang:system_info(dist_ctrl)],
176 104 [{connections, length(DistStats)} | merge_stats(DistStats)].
177
178 -spec get_up_time() -> {value, integer()}.
179 get_up_time() ->
180 103 {value, erlang:round(element(1, erlang:statistics(wall_clock))/1000)}.
181
182 -spec get_mnesia_running_db_nodes_count() -> {value, non_neg_integer()}.
183 get_mnesia_running_db_nodes_count() ->
184 105 {value, length(mnesia:system_info(running_db_nodes))}.
185
186 remove_host_type_metrics(HostType) ->
187 
:-(
 HostTypeName = get_host_type_prefix(HostType),
188 
:-(
 lists:foreach(fun remove_metric/1, exometer:find_entries([HostTypeName])).
189
190 remove_all_metrics() ->
191 83 persistent_term:erase(?PREFIXES),
192 83 lists:foreach(fun remove_metric/1, exometer:find_entries([])).
193
194 %% ---------------------------------------------------------------------
195 %% Internal functions
196 %% ---------------------------------------------------------------------
197
198 prepare_prefixes() ->
199 83 Prebuilt = maps:from_list([begin
200 426 Prefix = make_host_type_prefix(HT),
201 426 {Prefix, Prefix}
202 83 end || HT <- ?ALL_HOST_TYPES ]),
203 83 Prefixes = maps:from_list([ {HT, make_host_type_prefix(HT)}
204 83 || HT <- ?ALL_HOST_TYPES ]),
205 83 persistent_term:put(?PREFIXES, maps:merge(Prebuilt, Prefixes)).
206
207 -spec all_metrics_are_global() -> boolean().
208 all_metrics_are_global() ->
209 433307 mongoose_config:get_opt(all_metrics_are_global).
210
211 get_host_type_prefix(global) ->
212 151222 global;
213 get_host_type_prefix(HostType) when is_binary(HostType) ->
214 253372 case persistent_term:get(?PREFIXES, #{}) of
215 241939 #{HostType := HostTypePrefix} -> HostTypePrefix;
216 11433 #{} -> make_host_type_prefix(HostType)
217 end.
218
219 make_host_type_prefix(HT) when is_binary(HT) ->
220 12285 binary:replace(HT, <<" ">>, <<"_">>, [global]).
221
222 pick_prefix_by_all_metrics_are_global(HostType) ->
223 433307 case all_metrics_are_global() of
224 28800 true -> global;
225 404507 false -> get_host_type_prefix(HostType)
226 end.
227
228 pick_by_all_metrics_are_global(WhenGlobal, WhenNot) ->
229 
:-(
 case all_metrics_are_global() of
230 
:-(
 true -> WhenGlobal;
231 
:-(
 false -> WhenNot
232 end.
233
234 -spec name_by_all_metrics_are_global(HostType :: mongooseim:host_type_or_global(),
235 Name :: list()) -> FinalName :: list().
236 name_by_all_metrics_are_global(HostType, Name) ->
237 433307 [pick_prefix_by_all_metrics_are_global(HostType) | Name].
238
239 get_report_interval() ->
240 483 application:get_env(exometer_core, mongooseim_report_interval,
241 ?DEFAULT_REPORT_INTERVAL).
242
243 -spec do_create_generic_hook_metric(HostType :: mongooseim:host_type_or_global(),
244 Hook :: hook_name(),
245 UseOrSkip :: use_or_skip()) ->
246 ok | {ok, already_present} | {error, any()}.
247 do_create_generic_hook_metric(_, _, skip) ->
248 6826 ok;
249 do_create_generic_hook_metric(HostType, Hook, use) ->
250 9619 ensure_metric(HostType, Hook, spiral).
251
252 -spec do_increment_generic_hook_metric(HostType :: mongooseim:host_type_or_global(),
253 Hook :: hook_name(),
254 UseOrSkip :: use_or_skip()) ->
255 ok | {error, any()}.
256 do_increment_generic_hook_metric(_, _, skip) ->
257 92599 ok;
258 do_increment_generic_hook_metric(HostType, Hook, use) ->
259 92613 update(HostType, Hook, 1).
260
261 get_rdbms_stats(RDBMSWorkers) ->
262 
:-(
 RDBMSConnections = [{catch mongoose_rdbms:get_db_info(Pid), Pid} || Pid <- RDBMSWorkers],
263 
:-(
 Ports = [get_port_from_rdbms_connection(Conn) || Conn <- RDBMSConnections],
264 
:-(
 PortStats = [inet_stats(Port) || Port <- lists:flatten(Ports)],
265 
:-(
 [{workers, length(RDBMSConnections)} | merge_stats(PortStats)].
266
267 get_port_from_rdbms_connection({{ok, DB, Pid}, _WorkerPid}) when DB =:= mysql;
268 DB =:= pgsql ->
269 
:-(
 ProcState = sys:get_state(Pid),
270 
:-(
 get_port_from_proc_state(DB, ProcState);
271 get_port_from_rdbms_connection({{ok, odbc, Pid}, WorkerPid}) ->
272 
:-(
 Links = element(2, erlang:process_info(Pid, links)) -- [WorkerPid],
273 
:-(
 [Port || Port <- Links, is_port(Port), {name, "tcp_inet"} == erlang:port_info(Port, name)];
274 get_port_from_rdbms_connection(_) ->
275 
:-(
 undefined.
276
277 %% @doc Gets a socket from mysql/epgsql library Gen_server state
278 get_port_from_proc_state(mysql, State) ->
279 %% -record(state, {server_version, connection_id, socket, sockmod, ssl_opts,
280 %% host, port, user, password, log_warnings,
281 %% ping_timeout,
282 %% query_timeout, query_cache_time,
283 %% affected_rows = 0, status = 0, warning_count = 0, insert_id = 0,
284 %% transaction_level = 0, ping_ref = undefined,
285 %% stmts = dict:new(), query_cache = empty, cap_found_rows = false}).
286 
:-(
 SockInfo = element(4, State),
287 
:-(
 get_port_from_sock(SockInfo);
288 get_port_from_proc_state(pgsql, State) ->
289 %% -record(state, {mod,
290 %% sock,
291 %% data = <<>>,
292 %% backend,
293 %% handler,
294 %% codec,
295 %% queue = queue:new(),
296 %% async,
297 %% parameters = [],
298 %% types = [],
299 %% columns = [],
300 %% rows = [],
301 %% results = [],
302 %% batch = [],
303 %% sync_required,
304 %% txstatus,
305 %% complete_status :: undefined | atom() | {atom(), integer()},
306 %% repl_last_received_lsn,
307 %% repl_last_flushed_lsn,
308 %% repl_last_applied_lsn,
309 %% repl_feedback_required,
310 %% repl_cbmodule,
311 %% repl_cbstate,
312 %% repl_receiver}).
313 
:-(
 SockInfo = element(3, State),
314 
:-(
 get_port_from_sock(SockInfo).
315
316 get_port_from_sock({sslsocket, {_, Port, _, _}, _}) ->
317 
:-(
 Port;
318 get_port_from_sock(Port) ->
319 
:-(
 Port.
320
321 merge_stats(Stats) ->
322 104 OrdDict = lists:foldl(fun(Stat, Acc) ->
323 310 StatDict = orddict:from_list(Stat),
324 310 orddict:merge(fun merge_stats_fun/3, Acc, StatDict)
325 end, orddict:from_list(?EMPTY_INET_STATS), Stats),
326
327 104 orddict:to_list(OrdDict).
328
329 merge_stats_fun(recv_max, V1, V2) ->
330 310 erlang:max(V1, V2);
331 merge_stats_fun(send_max, V1, V2) ->
332 310 erlang:max(V1, V2);
333 merge_stats_fun(_, V1, V2) ->
334 1550 V1 + V2.
335
336 dist_inet_stats(Pid) when is_pid(Pid) ->
337 
:-(
 try
338 
:-(
 {ok, {sslsocket, FD, _Pids}} = tls_sender:dist_tls_socket(Pid),
339 
:-(
 gen_tcp = element(1, FD),
340 
:-(
 inet_stats(element(2, FD))
341 catch C:R:S ->
342 
:-(
 ?LOG_INFO(#{what => dist_inet_stats_failed, class => C, reason => R, stacktrace => S}),
343 
:-(
 ?EMPTY_INET_STATS
344 end;
345 dist_inet_stats(Port) ->
346 310 inet_stats(Port).
347
348 inet_stats(Port) ->
349 310 try
350 310 {ok, Stats} = inet:getstat(Port, ?INET_STATS),
351 310 Stats
352 catch C:R:S ->
353 
:-(
 ?LOG_INFO(#{what => inet_stats_failed, class => C, reason => R, stacktrace => S}),
354 
:-(
 ?EMPTY_INET_STATS
355 end.
356
357 remove_metric({Name, _, _}) ->
358 20024 exometer_admin:delete_entry(Name).
359
360 %% decided whether to use a metric for given hook or not
361 -spec filter_hook(hook_name()) -> use_or_skip().
362 3787 filter_hook(sm_register_connection_hook) -> skip;
363 3784 filter_hook(sm_remove_connection_hook) -> skip;
364 481 filter_hook(auth_failed) -> skip;
365 8502 filter_hook(user_send_packet) -> skip;
366 12164 filter_hook(user_receive_packet) -> skip;
367 462 filter_hook(xmpp_bounce_message) -> skip;
368 469 filter_hook(xmpp_stanza_dropped) -> skip;
369 30234 filter_hook(xmpp_send_element) -> skip;
370 532 filter_hook(roster_get) -> skip;
371 463 filter_hook(roster_set) -> skip;
372 810 filter_hook(roster_push) -> skip;
373 3376 filter_hook(register_user) -> skip;
374 3434 filter_hook(remove_user) -> skip;
375 492 filter_hook(privacy_iq_get) -> skip;
376 570 filter_hook(privacy_iq_set) -> skip;
377 27454 filter_hook(privacy_check_packet) -> skip;
378 47 filter_hook(mam_get_prefs) -> skip;
379 89 filter_hook(mam_set_prefs) -> skip;
380 187 filter_hook(mam_remove_archive) -> skip;
381 1397 filter_hook(mam_archive_message) -> skip;
382 1 filter_hook(mam_muc_get_prefs) -> skip;
383 1 filter_hook(mam_muc_set_prefs) -> skip;
384 205 filter_hook(mam_muc_remove_archive) -> skip;
385 82 filter_hook(mam_muc_lookup_messages) -> skip;
386 401 filter_hook(mam_muc_archive_message) -> skip;
387 1 filter_hook(mam_muc_flush_messages) -> skip;
388
389 102232 filter_hook(_) -> use.
390
391 -spec create_metrics(mongooseim:host_type()) -> 'ok'.
392 create_metrics(HostType) ->
393 426 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, spiral) end, ?GENERAL_SPIRALS),
394 426 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, counter) end, ?TOTAL_COUNTERS).
395
396 ensure_metric(HostType, Metric, Type, ShortType) when is_atom(Metric) ->
397 25507 ensure_metric(HostType, [Metric], Type, ShortType);
398
399 ensure_metric(HostType, Metric, Type, probe = ShortType) ->
400 455 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
401 455 {ShortType, Opts} = Type,
402 455 case exometer:info(PrefixedMetric, type) of
403 undefined ->
404 419 ExometerOpts = [{module, mongoose_metrics_probe}, {type, ShortType}] ++ Opts,
405 419 do_create_metric(PrefixedMetric, ad_hoc, ExometerOpts);
406 _ ->
407 36 {ok, already_present}
408 end;
409 ensure_metric(HostType, Metric, Type, ShortType) when is_list(Metric) ->
410 %% the split into ShortType and Type is needed because function metrics are
411 %% defined as tuples (that is Type), while exometer:info returns only 'function'
412 56930 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
413 56930 case exometer:info(PrefixedMetric, type) of
414 undefined ->
415 19759 do_create_metric(PrefixedMetric, Type, []);
416 37171 ShortType -> {ok, already_present}
417 end.
418
419 %% @doc Creates a metric and subcribes it to the reporters
420 -spec ensure_subscribed_metric(HostType :: mongooseim:host_type_or_global(),
421 Metric :: metric_name(),
422 Type :: metric_type()) -> ok | term().
423 ensure_subscribed_metric(HostType, Metric, Type) ->
424 15721 case ensure_metric(HostType, Metric, Type) of
425 ok ->
426 48 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
427 48 Reporters = exometer_report:list_reporters(),
428 48 Interval = get_report_interval(),
429 48 lists:foreach(
430 fun({Reporter, _Pid}) ->
431 
:-(
 FullMetric = {PrefixedMetric, Type, []},
432 
:-(
 subscribe_metric(Reporter, FullMetric, Interval)
433 end,
434 Reporters);
435 {ok, already_present} ->
436 15673 ?LOG_DEBUG(#{what => metric_already_present,
437 15673 host_type => HostType, metric => Metric, type => Type}),
438 15673 ok;
439 Other ->
440 
:-(
 ?LOG_WARNING(#{what => cannot_create_metric, reason => Other,
441 
:-(
 host_type => HostType, metric => Metric,type => Type}),
442 
:-(
 Other
443 end.
444
445 do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) ->
446 20178 case catch exometer:new(PrefixedMetric, ExometerType, ExometerOpts) of
447 2 {'EXIT', {exists, _}} -> {ok, already_present};
448 20176 ok -> ok;
449 
:-(
 {'EXIT', Error} -> {error, Error}
450 end.
451
452 create_data_metrics() ->
453 83 lists:foreach(fun(Metric) -> ensure_metric(global, Metric, histogram) end,
454 ?GLOBAL_HISTOGRAMS),
455 83 lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
456 ?DATA_FUN_METRICS).
457
458 start_metrics_subscriptions(Reporter, MetricPrefix, Interval) ->
459 
:-(
 [subscribe_metric(Reporter, Metric, Interval)
460 
:-(
 || Metric <- exometer:find_entries(MetricPrefix)].
461
462 subscribe_metric(Reporter, {Name, counter, _}, Interval) ->
463 
:-(
 subscribe_verbose(Reporter, Name, [value], Interval);
464 subscribe_metric(Reporter, {Name, histogram, _}, Interval) ->
465 
:-(
 subscribe_verbose(Reporter, Name, [min, mean, max, median, 95, 99, 999], Interval);
466 subscribe_metric(Reporter, {Name, _, _}, Interval) ->
467 
:-(
 subscribe_verbose(Reporter, Name, default, Interval).
468
469 subscribe_verbose(Reporter, Name, Types, Interval) ->
470 
:-(
 case exometer_report:subscribe(Reporter, Name, Types, Interval) of
471 
:-(
 ok -> ok;
472 Other ->
473 
:-(
 ?LOG_ERROR(#{what => metrics_subscribe_failed,
474 reporter => Reporter, metric_name => Name,
475 
:-(
 reason => Other}),
476 
:-(
 Other
477 end.
478
479 subscribe_to_all(Reporter, Interval) ->
480 
:-(
 HostTypePrefixes = pick_by_all_metrics_are_global([], ?ALL_HOST_TYPES),
481 
:-(
 lists:foreach(
482 fun(Prefix) ->
483 
:-(
 UnspacedPrefix = get_host_type_prefix(Prefix),
484 
:-(
 start_metrics_subscriptions(Reporter, [UnspacedPrefix], Interval)
485 end, [global | HostTypePrefixes]).
Line Hits Source