1: -module(service_mongoose_system_metrics_SUITE).
2:
3: -compile([export_all, nowarn_export_all]).
4:
5: -include_lib("common_test/include/ct.hrl").
6: -include_lib("eunit/include/eunit.hrl").
7:
8: -define(SERVER_URL, "http://localhost:8765").
9: -define(ETS_TABLE, qs).
10: -define(TRACKING_ID, "UA-151671255-2").
11: -define(TRACKING_ID_CI, "UA-151671255-1").
12: -define(TRACKING_ID_EXTRA, "UA-EXTRA-TRACKING-ID").
13:
14: -record(event, {
15: cid = "",
16: tid = "",
17: ec = "",
18: ea = "",
19: ev = "",
20: el = "" }).
21:
22: -import(distributed_helper, [mim/0, mim2/0, mim3/0, rpc/4,
23: require_rpc_nodes/1
24: ]).
25:
26: -import(component_helper, [connect_component/1,
27: disconnect_component/2,
28: spec/2,
29: common/1]).
30:
31: -import(domain_helper, [host_type/0]).
32: -import(config_parser_helper, [mod_config/2, config/2]).
33:
34: suite() ->
35: require_rpc_nodes([mim]).
36:
37: all() ->
38: [
39: system_metrics_are_not_reported_when_not_allowed,
40: periodic_report_available,
41: all_clustered_mongooses_report_the_same_client_id,
42: system_metrics_are_reported_to_google_analytics_when_mim_starts,
43: system_metrics_are_reported_to_configurable_google_analytics,
44: system_metrics_are_reported_to_a_json_file,
45: mongoose_version_is_reported,
46: cluster_uptime_is_reported,
47: xmpp_components_are_reported,
48: api_are_reported,
49: transport_mechanisms_are_reported,
50: outgoing_pools_are_reported,
51: xmpp_stanzas_counts_are_reported,
52: config_type_is_reported,
53: {group, module_opts},
54: {group, log_transparency}
55: ].
56:
57: groups() ->
58: [
59: {module_opts, [], [
60: module_opts_are_reported,
61: rdbms_module_opts_are_reported
62: ]},
63: {log_transparency, [], [
64: just_removed_from_config_logs_question,
65: in_config_unmodified_logs_request_for_agreement,
66: in_config_with_explicit_no_report_goes_off_silently,
67: in_config_with_explicit_reporting_goes_on_silently
68: ]}
69: ].
70:
71: -define(APPS, [inets, crypto, ssl, ranch, cowlib, cowboy]).
72:
73: %%--------------------------------------------------------------------
74: %% Suite configuration
75: %%--------------------------------------------------------------------
76: init_per_suite(Config) ->
77: [ {ok, _} = application:ensure_all_started(App) || App <- ?APPS ],
78: http_helper:start(8765, "/[...]", fun handler_init/1),
79: Config1 = escalus:init_per_suite(Config),
80: Config2 = dynamic_services:save_services([mim(), mim2()], Config1),
81: ejabberd_node_utils:init(Config2).
82:
83: end_per_suite(Config) ->
84: http_helper:stop(),
85: dynamic_services:restore_services(Config),
86: escalus:end_per_suite(Config).
87:
88: %%--------------------------------------------------------------------
89: %% Init & teardown
90: %%--------------------------------------------------------------------
91: init_per_group(module_opts, Config) ->
92: dynamic_modules:save_modules(host_type(), Config);
93: init_per_group(log_transparency, Config) ->
94: logger_ct_backend:start(),
95: logger_ct_backend:capture(warning),
96: Config;
97: init_per_group(_GroupName, Config) ->
98: Config.
99:
100: end_per_group(module_opts, Config) ->
101: dynamic_modules:restore_modules(Config);
102: end_per_group(log_transparency, Config) ->
103: logger_ct_backend:stop_capture(),
104: Config;
105: end_per_group(_GroupName, Config) ->
106: Config.
107:
108: init_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) ->
109: create_events_collection(),
110: disable_system_metrics(mim()),
111: delete_prev_client_id(mim()),
112: Config;
113: init_per_testcase(all_clustered_mongooses_report_the_same_client_id, Config) ->
114: create_events_collection(),
115: distributed_helper:add_node_to_cluster(mim2(), Config),
116: enable_system_metrics(mim()),
117: enable_system_metrics(mim2()),
118: Config;
119: init_per_testcase(system_metrics_are_reported_to_configurable_google_analytics, Config) ->
120: create_events_collection(),
121: enable_system_metrics_with_configurable_tracking_id(mim()),
122: Config;
123: init_per_testcase(xmpp_components_are_reported, Config) ->
124: create_events_collection(),
125: Config1 = get_components(common(Config), Config),
126: enable_system_metrics(mim()),
127: Config1;
128: init_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) ->
129: create_events_collection(),
130: enable_system_metrics(mim()),
131: Config1 = escalus:create_users(Config, escalus:get_users([alice, bob])),
132: escalus:init_per_testcase(CN, Config1);
133: init_per_testcase(rdbms_module_opts_are_reported = CN, Config) ->
134: case mongoose_helper:is_rdbms_enabled(host_type()) of
135: false ->
136: {skip, "RDBMS is not available"};
137: true ->
138: create_events_collection(),
139: dynamic_modules:ensure_modules(host_type(), required_modules(CN)),
140: enable_system_metrics(mim()),
141: Config
142: end;
143: init_per_testcase(module_opts_are_reported = CN, Config) ->
144: create_events_collection(),
145: dynamic_modules:ensure_modules(host_type(), required_modules(CN)),
146: enable_system_metrics(mim()),
147: Config;
148: init_per_testcase(_TestcaseName, Config) ->
149: create_events_collection(),
150: enable_system_metrics(mim()),
151: Config.
152:
153: end_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) ->
154: clear_events_collection(),
155: delete_prev_client_id(mim()),
156: Config;
157: end_per_testcase(all_clustered_mongooses_report_the_same_client_id , Config) ->
158: clear_events_collection(),
159: delete_prev_client_id(mim()),
160: Nodes = [mim(), mim2()],
161: [ begin delete_prev_client_id(Node), disable_system_metrics(Node) end || Node <- Nodes ],
162: distributed_helper:remove_node_from_cluster(mim2(), Config),
163: Config;
164: end_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) ->
165: clear_events_collection(),
166: disable_system_metrics(mim()),
167: escalus:delete_users(Config, escalus:get_users([alice, bob])),
168: escalus:end_per_testcase(CN, Config);
169: end_per_testcase(_TestcaseName, Config) ->
170: clear_events_collection(),
171: disable_system_metrics(mim()),
172: delete_prev_client_id(mim()),
173: Config.
174:
175:
176: %%--------------------------------------------------------------------
177: %% Tests
178: %%--------------------------------------------------------------------
179: system_metrics_are_not_reported_when_not_allowed(_Config) ->
180: true = system_metrics_service_is_disabled(mim()).
181:
182: periodic_report_available(_Config) ->
183: ReportsNumber = get_events_collection_size(),
184: mongoose_helper:wait_until(
185: fun() ->
186: NewReportsNumber = get_events_collection_size(),
187: NewReportsNumber > ReportsNumber + 1
188: end,
189: true).
190:
191: all_clustered_mongooses_report_the_same_client_id(_Config) ->
192: mongoose_helper:wait_until(fun is_host_count_reported/0, true),
193: all_event_have_the_same_client_id().
194:
195: system_metrics_are_reported_to_google_analytics_when_mim_starts(_Config) ->
196: mongoose_helper:wait_until(fun is_host_count_reported/0, true),
197: mongoose_helper:wait_until(fun are_modules_reported/0, true),
198: events_are_reported_to_primary_tracking_id(),
199: all_event_have_the_same_client_id().
200:
201: system_metrics_are_reported_to_configurable_google_analytics(_Config) ->
202: mongoose_helper:wait_until(fun is_host_count_reported/0, true),
203: mongoose_helper:wait_until(fun are_modules_reported/0, true),
204: events_are_reported_to_both_tracking_ids(),
205: all_event_have_the_same_client_id().
206:
207: system_metrics_are_reported_to_a_json_file(_Config) ->
208: ReportFilePath = rpc(mim(), mongoose_system_metrics_file, location, []),
209: ReportLastModified = rpc(mim(), filelib, last_modified, [ReportFilePath]),
210: Fun = fun() ->
211: ReportLastModified < rpc(mim(), filelib, last_modified, [ReportFilePath])
212: end,
213: mongoose_helper:wait_until(Fun, true),
214: %% now we read the content of the file and check if it's a valid JSON
215: {ok, File} = rpc(mim(), file, read_file, [ReportFilePath]),
216: jiffy:decode(File).
217:
218: module_opts_are_reported(_Config) ->
219: mongoose_helper:wait_until(fun are_modules_reported/0, true),
220: Backend = mongoose_helper:mnesia_or_rdbms_backend(),
221: check_module_backend(mod_bosh, mnesia),
222: check_module_backend(mod_event_pusher, push),
223: check_module_backend(mod_event_pusher_push, Backend),
224: check_module_backend(mod_http_upload, s3),
225: check_module_backend(mod_last, Backend),
226: check_module_backend(mod_muc, Backend),
227: check_module_backend(mod_muc_light, Backend),
228: check_module_backend(mod_offline, Backend),
229: check_module_backend(mod_privacy, Backend),
230: check_module_backend(mod_private, Backend),
231: check_module_backend(mod_pubsub, Backend),
232: check_module_opt(mod_push_service_mongoosepush, api_version, <<"\"v3\"">>),
233: check_module_backend(mod_roster, Backend),
234: check_module_backend(mod_vcard, Backend).
235:
236: rdbms_module_opts_are_reported(_Config) ->
237: mongoose_helper:wait_until(fun are_modules_reported/0, true),
238: check_module_backend(mod_auth_token, rdbms),
239: check_module_backend(mod_inbox, rdbms),
240: check_module_backend(mod_mam_meta, rdbms).
241:
242: check_module_backend(Module, Backend) ->
243: check_module_opt(Module, backend, atom_to_binary(Backend)).
244:
245: mongoose_version_is_reported(_Config) ->
246: mongoose_helper:wait_until(fun is_mongoose_version_reported/0, true).
247:
248: cluster_uptime_is_reported(_Config) ->
249: mongoose_helper:wait_until(fun is_cluster_uptime_reported/0, true).
250:
251: xmpp_components_are_reported(Config) ->
252: CompOpts = ?config(component1, Config),
253: {Component, Addr, _} = connect_component(CompOpts),
254: mongoose_helper:wait_until(fun are_xmpp_components_reported/0, true),
255: mongoose_helper:wait_until(fun more_than_one_component_is_reported/0, true),
256: disconnect_component(Component, Addr).
257:
258: api_are_reported(_Config) ->
259: mongoose_helper:wait_until(fun is_api_reported/0, true).
260:
261: transport_mechanisms_are_reported(_Config) ->
262: mongoose_helper:wait_until(fun are_transport_mechanisms_reported/0, true).
263:
264: outgoing_pools_are_reported(_Config) ->
265: mongoose_helper:wait_until(fun are_outgoing_pools_reported/0, true).
266:
267: xmpp_stanzas_counts_are_reported(Config) ->
268: escalus:story(Config, [{alice,1}, {bob,1}], fun(Alice, Bob) ->
269: mongoose_helper:wait_until(fun is_message_count_reported/0, true),
270: mongoose_helper:wait_until(fun is_iq_count_reported/0, true),
271: Sent = get_metric_value(<<"xmppMessageSent">>),
272: Received = get_metric_value(<<"xmppMessageReceived">>),
273: escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Hi">>)),
274: escalus:assert(is_chat_message, [<<"Hi">>], escalus:wait_for_stanza(Bob)),
275: F = fun() -> assert_message_count_is_incremented(Sent, Received) end,
276: mongoose_helper:wait_until(F, ok)
277: end).
278:
279: config_type_is_reported(_Config) ->
280: mongoose_helper:wait_until(fun is_config_type_reported/0, true).
281:
282: just_removed_from_config_logs_question(_Config) ->
283: disable_system_metrics(mim3()),
284: %% WHEN
285: Result = distributed_helper:rpc(
286: mim3(), service_mongoose_system_metrics, verify_if_configured, []),
287: %% THEN
288: ?assertEqual(ignore, Result).
289:
290: in_config_unmodified_logs_request_for_agreement(_Config) ->
291: %% WHEN
292: disable_system_metrics(mim()),
293: logger_ct_backend:capture(warning),
294: enable_system_metrics(mim()),
295: %% THEN
296: FilterFun = fun(_, Msg) ->
297: re:run(Msg, "MongooseIM docs", [global]) /= nomatch
298: end,
299: mongoose_helper:wait_until(fun() -> length(logger_ct_backend:recv(FilterFun)) end, 1),
300: %% CLEAN
301: logger_ct_backend:stop_capture(),
302: disable_system_metrics(mim()).
303:
304: in_config_with_explicit_no_report_goes_off_silently(_Config) ->
305: %% WHEN
306: logger_ct_backend:capture(warning),
307: start_system_metrics_service(mim(), #{report => false}),
308: logger_ct_backend:stop_capture(),
309: %% THEN
310: FilterFun = fun(warning, Msg) ->
311: re:run(Msg, "MongooseIM docs", [global]) /= nomatch;
312: (_,_) -> false
313: end,
314: [] = logger_ct_backend:recv(FilterFun),
315: %% CLEAN
316: disable_system_metrics(mim()).
317:
318: in_config_with_explicit_reporting_goes_on_silently(_Config) ->
319: %% WHEN
320: logger_ct_backend:capture(warning),
321: start_system_metrics_service(mim(), #{report => true}),
322: logger_ct_backend:stop_capture(),
323: %% THEN
324: FilterFun = fun(warning, Msg) ->
325: re:run(Msg, "MongooseIM docs", [global]) /= nomatch;
326: (_,_) -> false
327: end,
328: [] = logger_ct_backend:recv(FilterFun),
329: %% CLEAN
330: disable_system_metrics(mim()).
331:
332: %%--------------------------------------------------------------------
333: %% Helpers
334: %%--------------------------------------------------------------------
335:
336: required_modules(CaseName) ->
337: lists:filter(fun({Module, _Opts}) -> is_module_supported(Module) end,
338: modules_to_test(CaseName)).
339:
340: modules_to_test(module_opts_are_reported) ->
341: Backend = mongoose_helper:mnesia_or_rdbms_backend(),
342: [required_module(mod_bosh),
343: required_module(mod_event_pusher,
344: #{push => config([modules, mod_event_pusher, push], #{backend => Backend})}),
345: required_module(mod_http_upload, s3),
346: required_module(mod_last, Backend),
347: required_module(mod_muc, Backend),
348: required_module(mod_muc_light, Backend),
349: required_module(mod_offline, Backend),
350: required_module(mod_privacy, Backend),
351: required_module(mod_private, Backend),
352: required_module(mod_pubsub, Backend),
353: required_module(mod_push_service_mongoosepush),
354: required_module(mod_roster, Backend),
355: required_module(mod_vcard, Backend)];
356: modules_to_test(rdbms_module_opts_are_reported) ->
357: [required_module(mod_auth_token),
358: required_module(mod_inbox),
359: required_module(mod_mam_meta)].
360:
361: required_module(Module) ->
362: required_module(Module, #{}).
363:
364: required_module(Module, Backend) when is_atom(Backend) ->
365: {Module, mod_config(Module, #{backend => Backend})};
366: required_module(Module, Opts) ->
367: {Module, mod_config(Module, Opts)}.
368:
369: check_module_opt(Module, Key, Value) ->
370: case is_module_supported(Module) of
371: true ->
372: ?assertEqual(true, is_module_opt_reported(Module, Key, Value));
373: false ->
374: ct:log("Skipping unsupported module ~p", [Module])
375: end.
376:
377: is_module_supported(Module) ->
378: is_host_type_static() orelse supports_dynamic_domains(Module).
379:
380: is_host_type_static() ->
381: rpc(mim(), mongoose_domain_core, is_static, [host_type()]).
382:
383: supports_dynamic_domains(Module) ->
384: rpc(mim(), gen_mod, does_module_support, [Module, dynamic_domains]).
385:
386: all_event_have_the_same_client_id() ->
387: Tab = ets:tab2list(?ETS_TABLE),
388: UniqueSortedTab = lists:usort([Cid || #event{cid = Cid} <- Tab]),
389: 1 = length(UniqueSortedTab).
390:
391: is_host_count_reported() ->
392: is_in_table(<<"hosts">>).
393:
394: are_modules_reported() ->
395: is_in_table(<<"module">>).
396:
397: is_in_table(EventCategory) ->
398: Tab = ets:tab2list(?ETS_TABLE),
399: lists:any(
400: fun(#event{ec = EC}) ->
401: verify_category(EC, EventCategory)
402: end, Tab).
403:
404: verify_category(EC, <<"module">>) ->
405: Result = re:run(EC, "^mod_.*"),
406: case Result of
407: {match, _Captured} -> true;
408: nomatch -> false
409: end;
410: verify_category(EC, EC) ->
411: true;
412: verify_category(_EC, _EventCategory) ->
413: false.
414:
415: get_events_collection_size() ->
416: ets:info(?ETS_TABLE, size).
417:
418: enable_system_metrics(Node) ->
419: enable_system_metrics(Node, #{initial_report => 100, periodic_report => 100}).
420:
421: enable_system_metrics_with_configurable_tracking_id(Node) ->
422: enable_system_metrics(Node, #{initial_report => 100, periodic_report => 100,
423: tracking_id => ?TRACKING_ID_EXTRA}).
424:
425: enable_system_metrics(Node, Opts) ->
426: UrlArgs = [google_analytics_url, ?SERVER_URL],
427: ok = mongoose_helper:successful_rpc(Node, mongoose_config, set_opt, UrlArgs),
428: start_system_metrics_service(Node, Opts).
429:
430: start_system_metrics_service(Node, ExtraOpts) ->
431: Opts = config([services, service_mongoose_system_metrics], ExtraOpts),
432: dynamic_services:ensure_started(Node, service_mongoose_system_metrics, Opts).
433:
434: disable_system_metrics(Node) ->
435: dynamic_services:ensure_stopped(Node, service_mongoose_system_metrics),
436: mongoose_helper:successful_rpc(Node, mongoose_config, unset_opt, [ google_analytics_url ]).
437:
438: delete_prev_client_id(Node) ->
439: mongoose_helper:successful_rpc(Node, mnesia, delete_table, [service_mongoose_system_metrics]).
440:
441: create_events_collection() ->
442: ets:new(?ETS_TABLE, [duplicate_bag, named_table, public]).
443:
444: clear_events_collection() ->
445: ets:delete_all_objects(?ETS_TABLE).
446:
447: system_metrics_service_is_enabled(Node) ->
448: Pid = distributed_helper:rpc(Node, erlang, whereis, [service_mongoose_system_metrics]),
449: erlang:is_pid(Pid).
450:
451: system_metrics_service_is_disabled(Node) ->
452: not system_metrics_service_is_enabled(Node).
453:
454: events_are_reported_to_primary_tracking_id() ->
455: events_are_reported_to_tracking_ids([primary_tracking_id()]).
456:
457: events_are_reported_to_both_tracking_ids() ->
458: events_are_reported_to_tracking_ids([primary_tracking_id(), ?TRACKING_ID_EXTRA]).
459:
460: primary_tracking_id() ->
461: case os:getenv("CI") of
462: "true" -> ?TRACKING_ID_CI;
463: _ -> ?TRACKING_ID
464: end.
465:
466: events_are_reported_to_tracking_ids(ConfiguredTrackingIds) ->
467: Tab = ets:tab2list(?ETS_TABLE),
468: ActualTrackingIds = lists:usort([Tid || #event{tid = Tid} <- Tab]),
469: ExpectedTrackingIds = lists:sort([list_to_binary(Tid) || Tid <- ConfiguredTrackingIds]),
470: ?assertEqual(ExpectedTrackingIds, ActualTrackingIds).
471:
472: is_feature_reported(EventCategory, EventAction) ->
473: length(match_events(EventCategory, EventAction)) > 0.
474:
475: is_feature_reported(EventCategory, EventAction, EventLabel) ->
476: length(match_events(EventCategory, EventAction, EventLabel)) > 0.
477:
478: is_module_backend_reported(Module, Backend) ->
479: is_feature_reported(atom_to_binary(Module), <<"backend">>, atom_to_binary(Backend)).
480:
481: is_module_opt_reported(Module, Key, Value) ->
482: is_feature_reported(atom_to_binary(Module), atom_to_binary(Key), Value).
483:
484: is_mongoose_version_reported() ->
485: is_feature_reported(<<"cluster">>, <<"mim_version">>).
486:
487: is_cluster_uptime_reported() ->
488: is_feature_reported(<<"cluster">>, <<"uptime">>).
489:
490: are_xmpp_components_reported() ->
491: is_feature_reported(<<"cluster">>, <<"number_of_components">>).
492:
493: is_config_type_reported() ->
494: IsToml = is_feature_reported(<<"cluster">>, <<"config_type">>, <<"toml">>),
495: IsCfg = is_feature_reported(<<"cluster">>, <<"config_type">>, <<"cfg">>),
496: IsToml orelse IsCfg.
497:
498: is_api_reported() ->
499: is_in_table(<<"http_api">>).
500:
501: are_transport_mechanisms_reported() ->
502: is_in_table(<<"transport_mechanism">>).
503:
504: are_outgoing_pools_reported() ->
505: is_in_table(<<"outgoing_pools">>).
506:
507: is_iq_count_reported() ->
508: is_in_table(<<"xmppIqSent">>).
509:
510: is_message_count_reported() ->
511: is_in_table(<<"xmppMessageSent">>) andalso is_in_table(<<"xmppMessageReceived">>).
512:
513: assert_message_count_is_incremented(Sent, Received) ->
514: assert_increment(<<"xmppMessageSent">>, Sent),
515: assert_increment(<<"xmppMessageReceived">>, Received).
516:
517: assert_increment(EventCategory, InitialValue) ->
518: Events = match_events(EventCategory, integer_to_binary(InitialValue + 1), <<$1>>),
519: ?assertMatch([_], Events). % expect exactly one event with an increment of 1
520:
521: get_metric_value(EventCategory) ->
522: [#event{ea = Value} | _] = match_events(EventCategory),
523: binary_to_integer(Value).
524:
525: more_than_one_component_is_reported() ->
526: Events = match_events(<<"cluster">>, <<"number_of_components">>),
527: lists:any(fun(#event{el = EL}) ->
528: binary_to_integer(EL) > 0
529: end, Events).
530:
531: match_events(EC) ->
532: ets:match_object(?ETS_TABLE, #event{ec = EC, _ = '_'}).
533:
534: match_events(EC, EA) ->
535: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, _ = '_'}).
536:
537: match_events(EC, EA, EL) ->
538: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, el = EL, _ = '_'}).
539:
540: %%--------------------------------------------------------------------
541: %% Cowboy handlers
542: %%--------------------------------------------------------------------
543: handler_init(Req0) ->
544: {ok, Body, Req} = cowboy_req:read_body(Req0),
545: StrEvents = string:split(Body, "\n", all),
546: lists:map(
547: fun(StrEvent) ->
548: Event = str_to_event(StrEvent),
549: %% TODO there is a race condition when table is not available
550: ets:insert(?ETS_TABLE, Event)
551: end, StrEvents),
552: Req1 = cowboy_req:reply(200, #{}, <<"">>, Req),
553: {ok, Req1, no_state}.
554:
555: str_to_event(Qs) ->
556: StrParams = string:split(Qs, "&", all),
557: Params = lists:map(
558: fun(StrParam) ->
559: [StrKey, StrVal] = string:split(StrParam, "="),
560: {binary_to_atom(StrKey, utf8), StrVal}
561: end, StrParams),
562: #event{
563: cid = get_el(cid, Params),
564: tid = get_el(tid, Params),
565: ec = get_el(ec, Params),
566: ea = get_el(ea, Params),
567: el = get_el(el, Params),
568: ev = get_el(ev, Params)
569: }.
570:
571: get_el(Key, Proplist) ->
572: proplists:get_value(Key, Proplist, undef).
573:
574: get_components(Opts, Config) ->
575: Components = [component1, component2, vjud_component],
576: [ {C, Opts ++ spec(C, Config)} || C <- Components ] ++ Config.