./ct_report/coverage/mongoose_metrics.COVER.html

1 %%==============================================================================
2 %% Copyright 2014 Erlang Solutions Ltd.
3 %%
4 %% Licensed under the Apache License, Version 2.0 (the "License");
5 %% you may not use this file except in compliance with the License.
6 %% You may obtain a copy of the License at
7 %%
8 %% http://www.apache.org/licenses/LICENSE-2.0
9 %%
10 %% Unless required by applicable law or agreed to in writing, software
11 %% distributed under the License is distributed on an "AS IS" BASIS,
12 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 %% See the License for the specific language governing permissions and
14 %% limitations under the License.
15 %%==============================================================================
16 -module(mongoose_metrics).
17
18 -include("mongoose.hrl").
19 -include("mongoose_metrics_definitions.hrl").
20
21 %% API
22 -export([init/0,
23 create_global_metrics/0,
24 init_predefined_host_type_metrics/1,
25 init_subscriptions/0,
26 make_host_type_name/1,
27 create_generic_hook_metric/2,
28 ensure_db_pool_metric/1,
29 update/3,
30 ensure_metric/3,
31 ensure_subscribed_metric/3,
32 get_metric_value/1,
33 get_metric_values/1,
34 get_metric_value/2,
35 get_host_type_metric_names/1,
36 get_global_metric_names/0,
37 get_aggregated_values/1,
38 increment_generic_hook_metric/2,
39 get_rdbms_data_stats/0,
40 get_rdbms_data_stats/1,
41 get_dist_data_stats/0,
42 get_up_time/0,
43 get_mnesia_running_db_nodes_count/0,
44 remove_host_type_metrics/1,
45 remove_all_metrics/0,
46 get_report_interval/0
47 ]).
48
49 -ignore_xref([create_global_metrics/0, get_dist_data_stats/0, get_mnesia_running_db_nodes_count/0,
50 get_rdbms_data_stats/0, get_rdbms_data_stats/1, get_up_time/0,
51 init_subscriptions/0, make_host_type_name/1, remove_host_type_metrics/1,
52 get_report_interval/0]).
53
54 -define(DEFAULT_REPORT_INTERVAL, 60000). %%60s
55
56 -type use_or_skip() :: use | skip.
57 -type hook_name() :: atom().
58 -type metric_name() :: atom() | list(atom() | binary()).
59 -type short_metric_type() :: spiral | histogram | counter | gauge.
60 -type metric_type() :: tuple() | short_metric_type().
61 -type host_type_or_global() :: mongooseim:host_type() | global.
62
63 %% ---------------------------------------------------------------------
64 %% API
65 %% ---------------------------------------------------------------------
66
67 -spec init() -> ok.
68 init() ->
69 82 create_global_metrics(),
70 82 lists:foreach(
71 fun(HostType) ->
72 421 mongoose_metrics:init_predefined_host_type_metrics(HostType)
73 end, ?ALL_HOST_TYPES),
74 82 init_subscriptions().
75
76 create_global_metrics() ->
77 82 lists:foreach(fun({Metric, FunSpec, DataPoints}) ->
78 164 FunSpecTuple = list_to_tuple(FunSpec ++ [DataPoints]),
79 164 catch ensure_metric(global, Metric, FunSpecTuple)
80 end, ?VM_STATS),
81 82 lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
82 ?GLOBAL_COUNTERS),
83 82 create_data_metrics().
84
85 -spec init_predefined_host_type_metrics(mongooseim:host_type()) -> ok.
86 init_predefined_host_type_metrics(HostType) ->
87 421 create_metrics(HostType),
88 421 Hooks = mongoose_metrics_hooks:get_hooks(HostType),
89 421 ejabberd_hooks:add(Hooks),
90 421 ok.
91
92 init_subscriptions() ->
93 82 Reporters = exometer_report:list_reporters(),
94 82 lists:foreach(
95 fun({Name, _ReporterPid}) ->
96 
:-(
 Interval = get_report_interval(),
97 
:-(
 subscribe_to_all(Name, Interval)
98 end, Reporters).
99
100 -spec create_generic_hook_metric(mongooseim:host_type(), atom()) ->
101 ok | {ok, already_present} | {error, any()}.
102 create_generic_hook_metric(HostType, Hook) ->
103 16228 UseOrSkip = filter_hook(Hook),
104 16228 do_create_generic_hook_metric(HostType, Hook, UseOrSkip).
105
106 % TODO: change to HostType after mongoose_wpool_rdbms
107 ensure_db_pool_metric({rdbms, Host, Tag} = Name) ->
108 
:-(
 ensure_metric(Host,
109 [data, rdbms, Tag],
110 {function, mongoose_metrics, get_rdbms_data_stats, [[Name]], proplist,
111 [workers | ?INET_STATS]}).
112
113 -spec update(HostType :: mongooseim:host_type() | global, Name :: term() | list(),
114 Change :: term()) -> any().
115 update(HostType, Name, Change) when is_list(Name) ->
116 283961 exometer:update(name_by_all_metrics_are_global(HostType, Name), Change);
117 update(HostType, Name, Change) ->
118 168042 update(HostType, [Name], Change).
119
120 -spec ensure_metric(host_type_or_global(), metric_name(), metric_type()) ->
121 ok | {ok, already_present} | {error, any()}.
122 ensure_metric(HostType, Metric, Type) when is_tuple(Type) ->
123 863 ensure_metric(HostType, Metric, Type, element(1, Type));
124 ensure_metric(HostType, Metric, Type) ->
125 54684 ensure_metric(HostType, Metric, Type, Type).
126
127 get_metric_value(HostType, Name) when is_list(Name) ->
128 166 get_metric_value(name_by_all_metrics_are_global(HostType, Name));
129 get_metric_value(HostType, Name) ->
130 163 get_metric_value(HostType, [Name]).
131
132 get_metric_value(Metric) ->
133 297 exometer:get_value(Metric).
134
135 get_metric_values(Metric) when is_list(Metric) ->
136 58 exometer:get_values(Metric);
137 get_metric_values(HostType) ->
138 176 exometer:get_values([HostType]).
139
140 get_host_type_metric_names(HostType) ->
141 87 HostTypeName = make_host_type_name(HostType),
142 87 [MetricName || {[_HostTypeName | MetricName], _, _} <- exometer:find_entries([HostTypeName])].
143
144 get_global_metric_names() ->
145 2 get_host_type_metric_names(global).
146
147 get_aggregated_values(Metric) ->
148 6557 exometer:aggregate([{{['_', Metric], '_', '_'}, [], [true]}], [one, count, value]).
149
150 -spec increment_generic_hook_metric(mongooseim:host_type(), atom()) -> ok | {error, any()}.
151 increment_generic_hook_metric(HostType, Hook) ->
152 145934 UseOrSkip = filter_hook(Hook),
153 145934 do_increment_generic_hook_metric(HostType, Hook, UseOrSkip).
154
155 get_rdbms_data_stats() ->
156 
:-(
 Pools = lists:filter(fun({Type, _Host, _Tag}) -> Type == rdbms end, mongoose_wpool:get_pools()),
157 
:-(
 get_rdbms_data_stats(Pools).
158
159 get_rdbms_data_stats(Pools) ->
160 
:-(
 RDBMSWorkers =
161 lists:flatmap(
162 fun({Type, Host, Tag}) ->
163 
:-(
 PoolName = mongoose_wpool:make_pool_name(Type, Host, Tag),
164 
:-(
 Wpool = wpool_pool:find_wpool(PoolName),
165 
:-(
 PoolSize = wpool_pool:wpool_get(size, Wpool),
166 
:-(
 [whereis(wpool_pool:worker_name(PoolName, I)) || I <- lists:seq(1, PoolSize)]
167 end,
168 Pools),
169
170 
:-(
 get_rdbms_stats(RDBMSWorkers).
171
172 get_dist_data_stats() ->
173 94 DistStats = [inet_stats(Port) || {_, Port} <- erlang:system_info(dist_ctrl)],
174 94 [{connections, length(DistStats)} | merge_stats(DistStats)].
175
176 -spec get_up_time() -> {value, integer()}.
177 get_up_time() ->
178 94 {value, erlang:round(element(1, erlang:statistics(wall_clock))/1000)}.
179
180 -spec get_mnesia_running_db_nodes_count() -> {value, non_neg_integer()}.
181 get_mnesia_running_db_nodes_count() ->
182 96 {value, length(mnesia:system_info(running_db_nodes))}.
183
184 remove_host_type_metrics(HostType) ->
185 
:-(
 HostTypeName = make_host_type_name(HostType),
186 
:-(
 lists:foreach(fun remove_metric/1, exometer:find_entries([HostTypeName])).
187
188 remove_all_metrics() ->
189 82 lists:foreach(fun remove_metric/1, exometer:find_entries([])).
190
191 %% ---------------------------------------------------------------------
192 %% Internal functions
193 %% ---------------------------------------------------------------------
194
195 -spec all_metrics_are_global() -> boolean().
196 all_metrics_are_global() ->
197 339722 mongoose_config:get_opt(all_metrics_are_global).
198
199 pick_by_all_metrics_are_global(WhenGlobal, WhenNot) ->
200 339722 case all_metrics_are_global() of
201 28376 true -> WhenGlobal;
202 311346 false -> WhenNot
203 end.
204
205 -spec name_by_all_metrics_are_global(HostType :: mongooseim:host_type() | global,
206 Name :: list()) -> FinalName :: list().
207 name_by_all_metrics_are_global(HostType, Name) ->
208 339722 pick_by_all_metrics_are_global([global | Name], [make_host_type_name(HostType) | Name]).
209
210 get_report_interval() ->
211 48 application:get_env(exometer_core, mongooseim_report_interval,
212 ?DEFAULT_REPORT_INTERVAL).
213
214 -spec do_create_generic_hook_metric(HostType :: mongooseim:host_type() | global,
215 Hook :: hook_name(),
216 UseOrSkip :: use_or_skip()) ->
217 ok | {ok, already_present} | {error, any()}.
218 do_create_generic_hook_metric(_, _, skip) ->
219 6736 ok;
220 do_create_generic_hook_metric(HostType, Hook, use) ->
221 9492 ensure_metric(HostType, Hook, spiral).
222
223 -spec do_increment_generic_hook_metric(HostType :: mongooseim:host_type() | global,
224 Hook :: hook_name(),
225 UseOrSkip :: use_or_skip()) ->
226 ok | {error, any()}.
227 do_increment_generic_hook_metric(_, _, skip) ->
228 70393 ok;
229 do_increment_generic_hook_metric(HostType, Hook, use) ->
230 75541 update(HostType, Hook, 1).
231
232 get_rdbms_stats(RDBMSWorkers) ->
233 
:-(
 RDBMSConnections = [{catch mongoose_rdbms:get_db_info(Pid), Pid} || Pid <- RDBMSWorkers],
234 
:-(
 Ports = [get_port_from_rdbms_connection(Conn) || Conn <- RDBMSConnections],
235 
:-(
 PortStats = [inet_stats(Port) || Port <- lists:flatten(Ports)],
236 
:-(
 [{workers, length(RDBMSConnections)} | merge_stats(PortStats)].
237
238 get_port_from_rdbms_connection({{ok, DB, Pid}, _WorkerPid}) when DB =:= mysql;
239 DB =:= pgsql ->
240 
:-(
 ProcState = sys:get_state(Pid),
241 
:-(
 get_port_from_proc_state(DB, ProcState);
242 get_port_from_rdbms_connection({{ok, odbc, Pid}, WorkerPid}) ->
243 
:-(
 Links = element(2, erlang:process_info(Pid, links)) -- [WorkerPid],
244 
:-(
 [Port || Port <- Links, is_port(Port), {name, "tcp_inet"} == erlang:port_info(Port, name)];
245 get_port_from_rdbms_connection(_) ->
246 
:-(
 undefined.
247
248 %% @doc Gets a socket from mysql/epgsql library Gen_server state
249 get_port_from_proc_state(mysql, State) ->
250 %% -record(state, {server_version, connection_id, socket, sockmod, ssl_opts,
251 %% host, port, user, password, log_warnings,
252 %% ping_timeout,
253 %% query_timeout, query_cache_time,
254 %% affected_rows = 0, status = 0, warning_count = 0, insert_id = 0,
255 %% transaction_level = 0, ping_ref = undefined,
256 %% stmts = dict:new(), query_cache = empty, cap_found_rows = false}).
257 
:-(
 SockInfo = element(4, State),
258 
:-(
 get_port_from_sock(SockInfo);
259 get_port_from_proc_state(pgsql, State) ->
260 %% -record(state, {mod,
261 %% sock,
262 %% data = <<>>,
263 %% backend,
264 %% handler,
265 %% codec,
266 %% queue = queue:new(),
267 %% async,
268 %% parameters = [],
269 %% types = [],
270 %% columns = [],
271 %% rows = [],
272 %% results = [],
273 %% batch = [],
274 %% sync_required,
275 %% txstatus,
276 %% complete_status :: undefined | atom() | {atom(), integer()},
277 %% repl_last_received_lsn,
278 %% repl_last_flushed_lsn,
279 %% repl_last_applied_lsn,
280 %% repl_feedback_required,
281 %% repl_cbmodule,
282 %% repl_cbstate,
283 %% repl_receiver}).
284 
:-(
 SockInfo = element(3, State),
285 
:-(
 get_port_from_sock(SockInfo).
286
287 get_port_from_sock({sslsocket, {_, Port, _, _}, _}) ->
288 
:-(
 Port;
289 get_port_from_sock(Port) ->
290 
:-(
 Port.
291
292 merge_stats(Stats) ->
293 94 OrdDict = lists:foldl(fun(Stat, Acc) ->
294 280 StatDict = orddict:from_list(Stat),
295 280 orddict:merge(fun merge_stats_fun/3, Acc, StatDict)
296 end, orddict:from_list(?EMPTY_INET_STATS), Stats),
297
298 94 orddict:to_list(OrdDict).
299
300 merge_stats_fun(recv_max, V1, V2) ->
301 280 erlang:max(V1, V2);
302 merge_stats_fun(send_max, V1, V2) ->
303 280 erlang:max(V1, V2);
304 merge_stats_fun(_, V1, V2) ->
305 1400 V1 + V2.
306
307 inet_stats(Port) when is_port(Port) ->
308 280 {ok, Stats} = inet:getstat(Port, ?INET_STATS),
309 280 Stats;
310 %% TLS dist is operated by PID, not PORT directly, so we need to find the relevant port
311 inet_stats(Pid) when is_pid(Pid) ->
312 
:-(
 {links, Links} = erlang:process_info(Pid, links),
313 %% In case of TLS, controlling process of a TCP port is one of the linked proceses
314 %% so we should check all links of all processes linked to givan one
315 
:-(
 RelatedPidsAndPorts =
316 lists:map(fun(LinkedPid) ->
317 
:-(
 {links, SubLinks} = erlang:process_info(LinkedPid, links),
318 
:-(
 SubLinks
319 end, [Pid | Links]),
320
321 
:-(
 PortsTCP = lists:filter(
322 fun(Link) ->
323 
:-(
 case Link of
324 Port when is_port(Port) ->
325 
:-(
 {name, "tcp_inet"} == erlang:port_info(Port, name);
326 _ ->
327 
:-(
 false
328 end
329 end, lists:flatten(RelatedPidsAndPorts)),
330
331 
:-(
 case PortsTCP of
332 [Port | _] ->
333 
:-(
 inet_stats(Port);
334 _ ->
335 
:-(
 ?EMPTY_INET_STATS
336 end;
337 inet_stats(_) ->
338 
:-(
 ?EMPTY_INET_STATS.
339
340 remove_metric({Name, _, _}) ->
341 19733 exometer_admin:delete_entry(Name).
342
343 %% decided whether to use a metric for given hook or not
344 -spec filter_hook(hook_name()) -> use_or_skip().
345 3138 filter_hook(sm_register_connection_hook) -> skip;
346 3135 filter_hook(sm_remove_connection_hook) -> skip;
347 472 filter_hook(auth_failed) -> skip;
348 6815 filter_hook(user_send_packet) -> skip;
349 8949 filter_hook(user_receive_packet) -> skip;
350 450 filter_hook(xmpp_bounce_message) -> skip;
351 464 filter_hook(xmpp_stanza_dropped) -> skip;
352 23718 filter_hook(xmpp_send_element) -> skip;
353 530 filter_hook(roster_get) -> skip;
354 458 filter_hook(roster_set) -> skip;
355 763 filter_hook(roster_push) -> skip;
356 2736 filter_hook(register_user) -> skip;
357 2739 filter_hook(remove_user) -> skip;
358 487 filter_hook(privacy_iq_get) -> skip;
359 565 filter_hook(privacy_iq_set) -> skip;
360 21710 filter_hook(privacy_check_packet) -> skip;
361 
:-(
 filter_hook(mam_get_prefs) -> skip;
362 
:-(
 filter_hook(mam_set_prefs) -> skip;
363 
:-(
 filter_hook(mam_remove_archive) -> skip;
364 
:-(
 filter_hook(mam_archive_message) -> skip;
365 
:-(
 filter_hook(mam_muc_get_prefs) -> skip;
366 
:-(
 filter_hook(mam_muc_set_prefs) -> skip;
367 
:-(
 filter_hook(mam_muc_remove_archive) -> skip;
368 
:-(
 filter_hook(mam_muc_lookup_messages) -> skip;
369 
:-(
 filter_hook(mam_muc_archive_message) -> skip;
370 
:-(
 filter_hook(mam_muc_flush_messages) -> skip;
371
372 85033 filter_hook(_) -> use.
373
374 -spec create_metrics(mongooseim:host_type()) -> 'ok'.
375 create_metrics(HostType) ->
376 421 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, spiral) end, ?GENERAL_SPIRALS),
377 421 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, counter) end, ?TOTAL_COUNTERS).
378
379 ensure_metric(HostType, Metric, Type, ShortType) when is_atom(Metric) ->
380 24544 ensure_metric(HostType, [Metric], Type, ShortType);
381
382 ensure_metric(HostType, Metric, Type, probe = ShortType) ->
383 164 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
384 164 {ShortType, Opts} = Type,
385 164 case exometer:info(PrefixedMetric, type) of
386 undefined ->
387 164 ExometerOpts = [{module, mongoose_metrics_probe}, {type, ShortType}] ++ Opts,
388 164 do_create_metric(PrefixedMetric, ad_hoc, ExometerOpts);
389 _ ->
390 
:-(
 {ok, already_present}
391 end;
392 ensure_metric(HostType, Metric, Type, ShortType) when is_list(Metric) ->
393 %% the split into ShortType and Type is needed because function metrics are
394 %% defined as tuples (that is Type), while exometer:info returns only 'function'
395 55383 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
396 55383 case exometer:info(PrefixedMetric, type) of
397 undefined ->
398 19722 do_create_metric(PrefixedMetric, Type, []);
399 35661 ShortType -> {ok, already_present}
400 end.
401
402 %% @doc Creates a metric and subcribes it to the reporters
403 -spec ensure_subscribed_metric(HostType :: host_type_or_global(),
404 Metric :: metric_name(),
405 Type :: metric_type()) -> ok | term().
406 ensure_subscribed_metric(HostType, Metric, Type) ->
407 15716 case ensure_metric(HostType, Metric, Type) of
408 ok ->
409 48 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
410 48 Reporters = exometer_report:list_reporters(),
411 48 Interval = get_report_interval(),
412 48 lists:foreach(
413 fun({Reporter, _Pid}) ->
414 
:-(
 FullMetric = {PrefixedMetric, Type, []},
415 
:-(
 subscribe_metric(Reporter, FullMetric, Interval)
416 end,
417 Reporters);
418 {ok, already_present} ->
419 15668 ?LOG_DEBUG(#{what => metric_already_present,
420 15668 host_type => HostType, metric => Metric, type => Type}),
421 15668 ok;
422 Other ->
423 
:-(
 ?LOG_WARNING(#{what => cannot_create_metric, reason => Other,
424 
:-(
 host_type => HostType, metric => Metric,type => Type}),
425 
:-(
 Other
426 end.
427
428 do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) ->
429 19886 case catch exometer:new(PrefixedMetric, ExometerType, ExometerOpts) of
430 1 {'EXIT', {exists, _}} -> {ok, already_present};
431 19885 ok -> ok;
432 
:-(
 {'EXIT', Error} -> {error, Error}
433 end.
434
435 create_data_metrics() ->
436 82 lists:foreach(fun(Metric) -> ensure_metric(global, Metric, histogram) end,
437 ?GLOBAL_HISTOGRAMS),
438 82 lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
439 ?DATA_FUN_METRICS).
440
441 start_metrics_subscriptions(Reporter, MetricPrefix, Interval) ->
442 
:-(
 [subscribe_metric(Reporter, Metric, Interval)
443 
:-(
 || Metric <- exometer:find_entries(MetricPrefix)].
444
445 subscribe_metric(Reporter, {Name, counter, _}, Interval) ->
446 
:-(
 subscribe_verbose(Reporter, Name, [value], Interval);
447 subscribe_metric(Reporter, {Name, histogram, _}, Interval) ->
448 
:-(
 subscribe_verbose(Reporter, Name, [min, mean, max, median, 95, 99, 999], Interval);
449 subscribe_metric(Reporter, {Name, _, _}, Interval) ->
450 
:-(
 subscribe_verbose(Reporter, Name, default, Interval).
451
452 subscribe_verbose(Reporter, Name, Types, Interval) ->
453 
:-(
 case exometer_report:subscribe(Reporter, Name, Types, Interval) of
454 
:-(
 ok -> ok;
455 Other ->
456 
:-(
 ?LOG_ERROR(#{what => metrics_subscribe_failed,
457 reporter => Reporter, metric_name => Name,
458 
:-(
 reason => Other}),
459 
:-(
 Other
460 end.
461
462 subscribe_to_all(Reporter, Interval) ->
463 
:-(
 HostTypePrefixes = pick_by_all_metrics_are_global([], ?ALL_HOST_TYPES),
464 
:-(
 lists:foreach(
465 fun(Prefix) ->
466 
:-(
 UnspacedPrefix = make_host_type_name(Prefix),
467 
:-(
 start_metrics_subscriptions(Reporter, [UnspacedPrefix], Interval)
468 end, [global | HostTypePrefixes]).
469
470 make_host_type_name(HT) when is_atom(HT) ->
471 147579 HT;
472 make_host_type_name(HT) when is_binary(HT) ->
473 192244 binary:replace(HT, <<" ">>, <<"_">>, [global]).
Line Hits Source