./ct_report/coverage/mongoose_metrics.COVER.html

1 %%==============================================================================
2 %% Copyright 2014 Erlang Solutions Ltd.
3 %%
4 %% Licensed under the Apache License, Version 2.0 (the "License");
5 %% you may not use this file except in compliance with the License.
6 %% You may obtain a copy of the License at
7 %%
8 %% http://www.apache.org/licenses/LICENSE-2.0
9 %%
10 %% Unless required by applicable law or agreed to in writing, software
11 %% distributed under the License is distributed on an "AS IS" BASIS,
12 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 %% See the License for the specific language governing permissions and
14 %% limitations under the License.
15 %%==============================================================================
16 -module(mongoose_metrics).
17
18 -include("mongoose.hrl").
19 -include("mongoose_metrics_definitions.hrl").
20
21 %% API
22 -export([init/0,
23 init_mongooseim_metrics/0,
24 create_probe_metric/3,
25 ensure_db_pool_metric/1,
26 update/3,
27 ensure_metric/3,
28 ensure_subscribed_metric/3,
29 get_metric_value/1,
30 get_metric_values/1,
31 get_metric_value/2,
32 sample_metric/1,
33 get_host_type_metric_names/1,
34 get_global_metric_names/0,
35 get_aggregated_values/1,
36 get_rdbms_data_stats/0,
37 get_rdbms_data_stats/1,
38 get_dist_data_stats/0,
39 get_up_time/0,
40 get_mnesia_running_db_nodes_count/0,
41 remove_host_type_metrics/1,
42 remove_all_metrics/0,
43 get_report_interval/0
44 ]).
45
46 -ignore_xref([get_dist_data_stats/0, get_mnesia_running_db_nodes_count/0,
47 get_rdbms_data_stats/0, get_rdbms_data_stats/1, get_up_time/0,
48 remove_host_type_metrics/1, get_report_interval/0,
49 sample_metric/1, get_metric_value/1]).
50
51 -define(PREFIXES, mongoose_metrics_prefixes).
52 -define(DEFAULT_REPORT_INTERVAL, 60000). %%60s
53
54 -type metric_name() :: atom() | list(atom() | binary()).
55 -type short_metric_type() :: spiral | histogram | counter | gauge.
56 -type metric_type() :: tuple() | short_metric_type().
57
58 %% ---------------------------------------------------------------------
59 %% API
60 %% ---------------------------------------------------------------------
61
62 -spec init() -> ok.
63 init() ->
64 93 prepare_prefixes(),
65 93 create_vm_metrics(),
66 93 create_global_metrics(?GLOBAL_COUNTERS),
67 93 create_data_metrics(),
68 93 create_host_type_metrics().
69
70 -spec init_mongooseim_metrics() -> ok.
71 init_mongooseim_metrics() ->
72 93 create_host_type_hook_metrics(),
73 93 create_global_metrics(?MNESIA_COUNTERS),
74 93 init_subscriptions().
75
76 init_subscriptions() ->
77 93 Reporters = exometer_report:list_reporters(),
78 93 lists:foreach(
79 fun({Name, _ReporterPid}) ->
80 
:-(
 Interval = get_report_interval(),
81 
:-(
 subscribe_to_all(Name, Interval)
82 end, Reporters).
83
84 -spec create_probe_metric(mongooseim:host_type_or_global(), atom(), module()) ->
85 ok | {ok, already_present} | {error, any()}.
86 create_probe_metric(HostType, Name, Module) ->
87 279 {Metric, Spec} = ?PROBE(Name, Module),
88 279 ensure_metric(HostType, Metric, Spec).
89
90 % TODO: change to HostType after mongoose_wpool_rdbms
91 ensure_db_pool_metric({rdbms, Host, Tag} = Name) ->
92 95 ensure_metric(Host,
93 [data, rdbms, Tag],
94 {function, mongoose_metrics, get_rdbms_data_stats, [[Name]], proplist,
95 [workers | ?INET_STATS]}).
96
97 -spec update(HostType :: mongooseim:host_type_or_global(), Name :: term() | list(),
98 Change :: term()) -> any().
99 update(HostType, Name, Change) when is_list(Name) ->
100 577361 exometer:update(name_by_all_metrics_are_global(HostType, Name), Change);
101 update(HostType, Name, Change) ->
102 245154 update(HostType, [Name], Change).
103
104 -spec ensure_metric(mongooseim:host_type_or_global(), metric_name(), metric_type()) ->
105 ok | {ok, already_present} | {error, any()}.
106 ensure_metric(HostType, Metric, Type) when is_tuple(Type) ->
107 1031 ensure_metric(HostType, Metric, Type, element(1, Type));
108 ensure_metric(HostType, Metric, Type) ->
109 22064 ensure_metric(HostType, Metric, Type, Type).
110
111 get_metric_value(HostType, Name) when is_list(Name) ->
112 258 get_metric_value(name_by_all_metrics_are_global(HostType, Name));
113 get_metric_value(HostType, Name) ->
114 71 get_metric_value(HostType, [Name]).
115
116 get_metric_value(Metric) ->
117 318 exometer:get_value(Metric).
118
119 get_metric_values(Metric) when is_list(Metric) ->
120 152 exometer:get_values(Metric);
121 get_metric_values(HostType) ->
122 184 exometer:get_values([HostType]).
123
124 %% Force update a probe metric
125 sample_metric(Metric) ->
126 6 exometer:sample(Metric).
127
128 %% Return metrics that have simple values, i.e. that can be used with get_aggregated_values/1
129 get_host_type_metric_names(HostType) ->
130 91 HostTypeName = get_host_type_prefix(HostType),
131 91 [MetricName || {[_HostTypeName | MetricName], Type, _} <- exometer:find_entries([HostTypeName]),
132 9765 Type =:= gauge orelse Type =:= counter orelse Type =:= spiral].
133
134 get_global_metric_names() ->
135 2 get_host_type_metric_names(global).
136
137 get_aggregated_values(Metric) when is_list(Metric) ->
138 8092 exometer:aggregate([{{['_' | Metric], '_', '_'}, [], [true]}], [one, count, value]);
139 get_aggregated_values(Metric) when is_atom(Metric) ->
140 
:-(
 get_aggregated_values([Metric]).
141
142 get_rdbms_data_stats() ->
143 
:-(
 Pools = lists:filter(fun({Type, _Host, _Tag}) -> Type == rdbms end, mongoose_wpool:get_pools()),
144 
:-(
 get_rdbms_data_stats(Pools).
145
146 get_rdbms_data_stats(Pools) ->
147 130 RDBMSWorkers =
148 lists:flatmap(
149 fun({Type, Host, Tag}) ->
150 130 PoolName = mongoose_wpool:make_pool_name(Type, Host, Tag),
151 130 Wpool = wpool_pool:find_wpool(PoolName),
152 130 PoolSize = wpool_pool:wpool_get(size, Wpool),
153 130 [whereis(wpool_pool:worker_name(PoolName, I)) || I <- lists:seq(1, PoolSize)]
154 end,
155 Pools),
156
157 130 get_rdbms_stats(RDBMSWorkers).
158
159 get_dist_data_stats() ->
160 132 DistStats = [dist_inet_stats(PortOrPid) || {_, PortOrPid} <- erlang:system_info(dist_ctrl)],
161 132 [{connections, length(DistStats)} | merge_stats(DistStats)].
162
163 -spec get_up_time() -> {value, integer()}.
164 get_up_time() ->
165 130 {value, erlang:round(element(1, erlang:statistics(wall_clock))/1000)}.
166
167 -spec get_mnesia_running_db_nodes_count() -> {value, non_neg_integer()}.
168 get_mnesia_running_db_nodes_count() ->
169 131 {value, length(mnesia:system_info(running_db_nodes))}.
170
171 remove_host_type_metrics(HostType) ->
172 
:-(
 HostTypeName = get_host_type_prefix(HostType),
173 
:-(
 lists:foreach(fun remove_metric/1, exometer:find_entries([HostTypeName])).
174
175 remove_all_metrics() ->
176 93 persistent_term:erase(?PREFIXES),
177 93 lists:foreach(fun remove_metric/1, exometer:find_entries([])).
178
179 %% ---------------------------------------------------------------------
180 %% Internal functions
181 %% ---------------------------------------------------------------------
182
183 prepare_prefixes() ->
184 93 Prebuilt = maps:from_list([begin
185 508 Prefix = make_host_type_prefix(HT),
186 508 {Prefix, Prefix}
187 93 end || HT <- ?ALL_HOST_TYPES ]),
188 93 Prefixes = maps:from_list([ {HT, make_host_type_prefix(HT)}
189 93 || HT <- ?ALL_HOST_TYPES ]),
190 93 persistent_term:put(?PREFIXES, maps:merge(Prebuilt, Prefixes)).
191
192 -spec all_metrics_are_global() -> boolean().
193 all_metrics_are_global() ->
194 600714 mongoose_config:get_opt(all_metrics_are_global).
195
196 get_host_type_prefix(global) ->
197 258866 global;
198 get_host_type_prefix(HostType) when is_binary(HostType) ->
199 329676 case persistent_term:get(?PREFIXES, #{}) of
200 329675 #{HostType := HostTypePrefix} -> HostTypePrefix;
201 1 #{} -> make_host_type_prefix(HostType)
202 end.
203
204 make_host_type_prefix(HT) when is_binary(HT) ->
205 1017 binary:replace(HT, <<" ">>, <<"_">>, [global]).
206
207 pick_prefix_by_all_metrics_are_global(HostType) ->
208 600714 case all_metrics_are_global() of
209 12263 true -> global;
210 588451 false -> get_host_type_prefix(HostType)
211 end.
212
213 pick_by_all_metrics_are_global(WhenGlobal, WhenNot) ->
214 
:-(
 case all_metrics_are_global() of
215 
:-(
 true -> WhenGlobal;
216 
:-(
 false -> WhenNot
217 end.
218
219 -spec name_by_all_metrics_are_global(HostType :: mongooseim:host_type_or_global(),
220 Name :: list()) -> FinalName :: list().
221 name_by_all_metrics_are_global(HostType, Name) ->
222 600714 [pick_prefix_by_all_metrics_are_global(HostType) | Name].
223
224 get_report_interval() ->
225 471 application:get_env(exometer_core, mongooseim_report_interval,
226 ?DEFAULT_REPORT_INTERVAL).
227
228 get_rdbms_stats(RDBMSWorkers) ->
229 130 RDBMSConnections = [{catch mongoose_rdbms:get_db_info(Pid), Pid} || Pid <- RDBMSWorkers],
230 130 Ports = [get_port_from_rdbms_connection(Conn) || Conn <- RDBMSConnections],
231 130 PortStats = [inet_stats(Port) || Port <- lists:flatten(Ports)],
232 130 [{workers, length(RDBMSConnections)} | merge_stats(PortStats)].
233
234 get_port_from_rdbms_connection({{ok, DB, Pid}, _WorkerPid}) when DB =:= mysql;
235 DB =:= pgsql ->
236 
:-(
 ProcState = sys:get_state(Pid),
237 
:-(
 get_port_from_proc_state(DB, ProcState);
238 get_port_from_rdbms_connection({{ok, odbc, Pid}, WorkerPid}) ->
239 650 Links = element(2, erlang:process_info(Pid, links)) -- [WorkerPid],
240 650 [Port || Port <- Links, is_port(Port), {name, "tcp_inet"} == erlang:port_info(Port, name)];
241 get_port_from_rdbms_connection(_) ->
242 
:-(
 undefined.
243
244 %% @doc Gets a socket from mysql/epgsql library Gen_server state
245 get_port_from_proc_state(mysql, State) ->
246 %% -record(state, {server_version, connection_id, socket, sockmod, ssl_opts,
247 %% host, port, user, password, log_warnings,
248 %% ping_timeout,
249 %% query_timeout, query_cache_time,
250 %% affected_rows = 0, status = 0, warning_count = 0, insert_id = 0,
251 %% transaction_level = 0, ping_ref = undefined,
252 %% stmts = dict:new(), query_cache = empty, cap_found_rows = false}).
253 
:-(
 SockInfo = element(4, State),
254 
:-(
 get_port_from_sock(SockInfo);
255 get_port_from_proc_state(pgsql, State) ->
256 %% -record(state, {mod,
257 %% sock,
258 %% data = <<>>,
259 %% backend,
260 %% handler,
261 %% codec,
262 %% queue = queue:new(),
263 %% async,
264 %% parameters = [],
265 %% types = [],
266 %% columns = [],
267 %% rows = [],
268 %% results = [],
269 %% batch = [],
270 %% sync_required,
271 %% txstatus,
272 %% complete_status :: undefined | atom() | {atom(), integer()},
273 %% repl_last_received_lsn,
274 %% repl_last_flushed_lsn,
275 %% repl_last_applied_lsn,
276 %% repl_feedback_required,
277 %% repl_cbmodule,
278 %% repl_cbstate,
279 %% repl_receiver}).
280 
:-(
 SockInfo = element(3, State),
281 
:-(
 get_port_from_sock(SockInfo).
282
283 get_port_from_sock({sslsocket, {_, Port, _, _}, _}) ->
284 
:-(
 Port;
285 get_port_from_sock(Port) ->
286 
:-(
 Port.
287
288 merge_stats(Stats) ->
289 262 OrdDict = lists:foldl(fun(Stat, Acc) ->
290 1575 StatDict = orddict:from_list(Stat),
291 1575 orddict:merge(fun merge_stats_fun/3, Acc, StatDict)
292 end, orddict:from_list(?EMPTY_INET_STATS), Stats),
293
294 262 orddict:to_list(OrdDict).
295
296 merge_stats_fun(recv_max, V1, V2) ->
297 1575 erlang:max(V1, V2);
298 merge_stats_fun(send_max, V1, V2) ->
299 1575 erlang:max(V1, V2);
300 merge_stats_fun(_, V1, V2) ->
301 7875 V1 + V2.
302
303 dist_inet_stats(Pid) when is_pid(Pid) ->
304 
:-(
 try
305 
:-(
 {ok, {sslsocket, FD, _Pids}} = tls_sender:dist_tls_socket(Pid),
306 
:-(
 gen_tcp = element(1, FD),
307 
:-(
 inet_stats(element(2, FD))
308 catch C:R:S ->
309 
:-(
 ?LOG_INFO(#{what => dist_inet_stats_failed, class => C, reason => R, stacktrace => S}),
310 
:-(
 ?EMPTY_INET_STATS
311 end;
312 dist_inet_stats(Port) ->
313 275 inet_stats(Port).
314
315 inet_stats(Port) ->
316 1575 try
317 1575 {ok, Stats} = inet:getstat(Port, ?INET_STATS),
318 1575 Stats
319 catch C:R:S ->
320 
:-(
 ?LOG_INFO(#{what => inet_stats_failed, class => C, reason => R, stacktrace => S}),
321 
:-(
 ?EMPTY_INET_STATS
322 end.
323
324 remove_metric({Name, _, _}) ->
325 32002 exometer_admin:delete_entry(Name).
326
327 create_global_metrics(Metrics) ->
328 186 lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end, Metrics).
329
330 create_vm_metrics() ->
331 93 lists:foreach(fun({Metric, FunSpec, DataPoints}) ->
332 186 FunSpecTuple = list_to_tuple(FunSpec ++ [DataPoints]),
333 186 catch ensure_metric(global, Metric, FunSpecTuple)
334 end, ?VM_STATS).
335
336 -spec create_host_type_metrics() -> ok.
337 create_host_type_metrics() ->
338 93 lists:foreach(fun create_host_type_metrics/1, ?ALL_HOST_TYPES).
339
340 -spec create_host_type_metrics(mongooseim:host_type()) -> 'ok'.
341 create_host_type_metrics(HostType) ->
342 508 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, spiral) end, ?GENERAL_SPIRALS),
343 508 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, histogram) end, ?GENERAL_HISTOGRAMS),
344 508 lists:foreach(fun(Name) -> ensure_metric(HostType, Name, counter) end, ?TOTAL_COUNTERS).
345
346 -spec create_host_type_hook_metrics() -> ok.
347 create_host_type_hook_metrics() ->
348 93 lists:foreach(fun create_host_type_hook_metrics/1, ?ALL_HOST_TYPES).
349
350 -spec create_host_type_hook_metrics(mongooseim:host_type()) -> 'ok'.
351 create_host_type_hook_metrics(HostType) ->
352 508 Hooks = mongoose_metrics_hooks:get_hooks(HostType),
353 508 gen_hook:add_handlers(Hooks).
354
355 ensure_metric(HostType, Metric, Type, ShortType) when is_atom(Metric) ->
356 18109 ensure_metric(HostType, [Metric], Type, ShortType);
357
358 ensure_metric(HostType, Metric, Type, probe = ShortType) ->
359 471 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
360 471 {ShortType, Opts} = Type,
361 471 case exometer:info(PrefixedMetric, type) of
362 undefined ->
363 467 ExometerOpts = [{module, mongoose_metrics_probe}, {type, ShortType}] ++ Opts,
364 467 do_create_metric(PrefixedMetric, ad_hoc, ExometerOpts);
365 _ ->
366 4 {ok, already_present}
367 end;
368 ensure_metric(HostType, Metric, Type, ShortType) when is_list(Metric) ->
369 %% the split into ShortType and Type is needed because function metrics are
370 %% defined as tuples (that is Type), while exometer:info returns only 'function'
371 22624 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
372 22624 case exometer:info(PrefixedMetric, type) of
373 undefined ->
374 14434 do_create_metric(PrefixedMetric, Type, []);
375 8190 ShortType -> {ok, already_present}
376 end.
377
378 %% @doc Creates a metric and subcribes it to the reporters
379 -spec ensure_subscribed_metric(HostType :: mongooseim:host_type_or_global(),
380 Metric :: metric_name(),
381 Type :: metric_type()) -> ok | term().
382 ensure_subscribed_metric(HostType, Metric, Type) ->
383 
:-(
 case ensure_metric(HostType, Metric, Type) of
384 ok ->
385 
:-(
 PrefixedMetric = name_by_all_metrics_are_global(HostType, Metric),
386 
:-(
 Reporters = exometer_report:list_reporters(),
387 
:-(
 Interval = get_report_interval(),
388 
:-(
 lists:foreach(
389 fun({Reporter, _Pid}) ->
390 
:-(
 FullMetric = {PrefixedMetric, Type, []},
391 
:-(
 subscribe_metric(Reporter, FullMetric, Interval)
392 end,
393 Reporters);
394 {ok, already_present} ->
395 
:-(
 ?LOG_DEBUG(#{what => metric_already_present,
396 
:-(
 host_type => HostType, metric => Metric, type => Type}),
397 
:-(
 ok;
398 Other ->
399 
:-(
 ?LOG_WARNING(#{what => cannot_create_metric, reason => Other,
400 
:-(
 host_type => HostType, metric => Metric,type => Type}),
401 
:-(
 Other
402 end.
403
404 do_create_metric(PrefixedMetric, ExometerType, ExometerOpts) ->
405 14901 case catch exometer:new(PrefixedMetric, ExometerType, ExometerOpts) of
406 
:-(
 {'EXIT', {exists, _}} -> {ok, already_present};
407 14901 ok -> ok;
408 
:-(
 {'EXIT', Error} -> {error, Error}
409 end.
410
411 create_data_metrics() ->
412 93 lists:foreach(fun(Metric) -> ensure_metric(global, Metric, histogram) end,
413 ?GLOBAL_HISTOGRAMS),
414 93 lists:foreach(fun(Metric) -> ensure_metric(global, Metric, spiral) end,
415 ?GLOBAL_SPIRALS),
416 93 lists:foreach(fun({Metric, Spec}) -> ensure_metric(global, Metric, Spec) end,
417 ?DATA_FUN_METRICS).
418
419 start_metrics_subscriptions(Reporter, MetricPrefix, Interval) ->
420 
:-(
 [subscribe_metric(Reporter, Metric, Interval)
421 
:-(
 || Metric <- exometer:find_entries(MetricPrefix)].
422
423 subscribe_metric(Reporter, {Name, counter, _}, Interval) ->
424 
:-(
 subscribe_verbose(Reporter, Name, [value], Interval);
425 subscribe_metric(Reporter, {Name, histogram, _}, Interval) ->
426 
:-(
 subscribe_verbose(Reporter, Name, [min, mean, max, median, 95, 99, 999], Interval);
427 subscribe_metric(Reporter, {Name, _, _}, Interval) ->
428 
:-(
 subscribe_verbose(Reporter, Name, default, Interval).
429
430 subscribe_verbose(Reporter, Name, Types, Interval) ->
431 
:-(
 case exometer_report:subscribe(Reporter, Name, Types, Interval) of
432 
:-(
 ok -> ok;
433 Other ->
434 
:-(
 ?LOG_ERROR(#{what => metrics_subscribe_failed,
435 reporter => Reporter, metric_name => Name,
436 
:-(
 reason => Other}),
437 
:-(
 Other
438 end.
439
440 subscribe_to_all(Reporter, Interval) ->
441 
:-(
 HostTypePrefixes = pick_by_all_metrics_are_global([], ?ALL_HOST_TYPES),
442 
:-(
 lists:foreach(
443 fun(Prefix) ->
444 
:-(
 UnspacedPrefix = get_host_type_prefix(Prefix),
445 
:-(
 start_metrics_subscriptions(Reporter, [UnspacedPrefix], Interval)
446 end, [global | HostTypePrefixes]).
Line Hits Source