1: -module(service_mongoose_system_metrics_SUITE). 2: 3: -compile([export_all, nowarn_export_all]). 4: 5: -include_lib("common_test/include/ct.hrl"). 6: -include_lib("eunit/include/eunit.hrl"). 7: 8: -define(SERVER_URL, "http://localhost:8765"). 9: -define(ETS_TABLE, qs). 10: -define(TRACKING_ID, "UA-151671255-2"). 11: -define(TRACKING_ID_CI, "UA-151671255-1"). 12: -define(TRACKING_ID_EXTRA, "UA-EXTRA-TRACKING-ID"). 13: 14: -record(event, { 15: cid = "", 16: tid = "", 17: ec = "", 18: ea = "", 19: ev = "", 20: el = "" }). 21: 22: -import(distributed_helper, [mim/0, mim2/0, mim3/0, rpc/4, 23: require_rpc_nodes/1 24: ]). 25: 26: -import(component_helper, [connect_component/1, 27: disconnect_component/2, 28: spec/2, 29: common/1]). 30: 31: -import(domain_helper, [host_type/0]). 32: -import(config_parser_helper, [mod_config/2, config/2]). 33: 34: suite() -> 35: require_rpc_nodes([mim]). 36: 37: all() -> 38: [ 39: system_metrics_are_not_reported_when_not_allowed, 40: periodic_report_available, 41: all_clustered_mongooses_report_the_same_client_id, 42: system_metrics_are_reported_to_google_analytics_when_mim_starts, 43: system_metrics_are_reported_to_configurable_google_analytics, 44: system_metrics_are_reported_to_a_json_file, 45: mongoose_version_is_reported, 46: cluster_uptime_is_reported, 47: xmpp_components_are_reported, 48: api_are_reported, 49: transport_mechanisms_are_reported, 50: outgoing_pools_are_reported, 51: xmpp_stanzas_counts_are_reported, 52: config_type_is_reported, 53: {group, module_opts}, 54: {group, log_transparency} 55: ]. 56: 57: groups() -> 58: [ 59: {module_opts, [], [ 60: module_opts_are_reported, 61: rdbms_module_opts_are_reported 62: ]}, 63: {log_transparency, [], [ 64: just_removed_from_config_logs_question, 65: in_config_unmodified_logs_request_for_agreement, 66: in_config_with_explicit_no_report_goes_off_silently, 67: in_config_with_explicit_reporting_goes_on_silently 68: ]} 69: ]. 70: 71: -define(APPS, [inets, crypto, ssl, ranch, cowlib, cowboy]). 72: 73: %%-------------------------------------------------------------------- 74: %% Suite configuration 75: %%-------------------------------------------------------------------- 76: init_per_suite(Config) -> 77: [ {ok, _} = application:ensure_all_started(App) || App <- ?APPS ], 78: http_helper:start(8765, "/[...]", fun handler_init/1), 79: Config1 = escalus:init_per_suite(Config), 80: Config2 = dynamic_services:save_services([mim(), mim2()], Config1), 81: ejabberd_node_utils:init(Config2). 82: 83: end_per_suite(Config) -> 84: http_helper:stop(), 85: dynamic_services:restore_services(Config), 86: escalus:end_per_suite(Config). 87: 88: %%-------------------------------------------------------------------- 89: %% Init & teardown 90: %%-------------------------------------------------------------------- 91: init_per_group(module_opts, Config) -> 92: dynamic_modules:save_modules(host_type(), Config); 93: init_per_group(log_transparency, Config) -> 94: logger_ct_backend:start(), 95: logger_ct_backend:capture(warning), 96: Config; 97: init_per_group(_GroupName, Config) -> 98: Config. 99: 100: end_per_group(module_opts, Config) -> 101: dynamic_modules:restore_modules(Config); 102: end_per_group(log_transparency, Config) -> 103: logger_ct_backend:stop_capture(), 104: Config; 105: end_per_group(_GroupName, Config) -> 106: Config. 107: 108: init_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) -> 109: create_events_collection(), 110: disable_system_metrics(mim()), 111: delete_prev_client_id(mim()), 112: Config; 113: init_per_testcase(all_clustered_mongooses_report_the_same_client_id, Config) -> 114: create_events_collection(), 115: distributed_helper:add_node_to_cluster(mim2(), Config), 116: enable_system_metrics(mim()), 117: enable_system_metrics(mim2()), 118: Config; 119: init_per_testcase(system_metrics_are_reported_to_configurable_google_analytics, Config) -> 120: create_events_collection(), 121: enable_system_metrics_with_configurable_tracking_id(mim()), 122: Config; 123: init_per_testcase(xmpp_components_are_reported, Config) -> 124: create_events_collection(), 125: Config1 = get_components(common(Config), Config), 126: enable_system_metrics(mim()), 127: Config1; 128: init_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) -> 129: create_events_collection(), 130: enable_system_metrics(mim()), 131: Config1 = escalus:create_users(Config, escalus:get_users([alice, bob])), 132: escalus:init_per_testcase(CN, Config1); 133: init_per_testcase(rdbms_module_opts_are_reported = CN, Config) -> 134: case mongoose_helper:is_rdbms_enabled(host_type()) of 135: false -> 136: {skip, "RDBMS is not available"}; 137: true -> 138: create_events_collection(), 139: dynamic_modules:ensure_modules(host_type(), required_modules(CN)), 140: enable_system_metrics(mim()), 141: Config 142: end; 143: init_per_testcase(module_opts_are_reported = CN, Config) -> 144: create_events_collection(), 145: dynamic_modules:ensure_modules(host_type(), required_modules(CN)), 146: enable_system_metrics(mim()), 147: Config; 148: init_per_testcase(_TestcaseName, Config) -> 149: create_events_collection(), 150: enable_system_metrics(mim()), 151: Config. 152: 153: end_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) -> 154: clear_events_collection(), 155: delete_prev_client_id(mim()), 156: Config; 157: end_per_testcase(all_clustered_mongooses_report_the_same_client_id , Config) -> 158: clear_events_collection(), 159: delete_prev_client_id(mim()), 160: Nodes = [mim(), mim2()], 161: [ begin delete_prev_client_id(Node), disable_system_metrics(Node) end || Node <- Nodes ], 162: distributed_helper:remove_node_from_cluster(mim2(), Config), 163: Config; 164: end_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) -> 165: clear_events_collection(), 166: disable_system_metrics(mim()), 167: escalus:delete_users(Config, escalus:get_users([alice, bob])), 168: escalus:end_per_testcase(CN, Config); 169: end_per_testcase(_TestcaseName, Config) -> 170: clear_events_collection(), 171: disable_system_metrics(mim()), 172: delete_prev_client_id(mim()), 173: Config. 174: 175: 176: %%-------------------------------------------------------------------- 177: %% Tests 178: %%-------------------------------------------------------------------- 179: system_metrics_are_not_reported_when_not_allowed(_Config) -> 180: true = system_metrics_service_is_disabled(mim()). 181: 182: periodic_report_available(_Config) -> 183: ReportsNumber = get_events_collection_size(), 184: mongoose_helper:wait_until( 185: fun() -> 186: NewReportsNumber = get_events_collection_size(), 187: NewReportsNumber > ReportsNumber + 1 188: end, 189: true). 190: 191: all_clustered_mongooses_report_the_same_client_id(_Config) -> 192: mongoose_helper:wait_until(fun is_host_count_reported/0, true), 193: all_event_have_the_same_client_id(). 194: 195: system_metrics_are_reported_to_google_analytics_when_mim_starts(_Config) -> 196: mongoose_helper:wait_until(fun is_host_count_reported/0, true), 197: mongoose_helper:wait_until(fun are_modules_reported/0, true), 198: events_are_reported_to_primary_tracking_id(), 199: all_event_have_the_same_client_id(). 200: 201: system_metrics_are_reported_to_configurable_google_analytics(_Config) -> 202: mongoose_helper:wait_until(fun is_host_count_reported/0, true), 203: mongoose_helper:wait_until(fun are_modules_reported/0, true), 204: events_are_reported_to_both_tracking_ids(), 205: all_event_have_the_same_client_id(). 206: 207: system_metrics_are_reported_to_a_json_file(_Config) -> 208: ReportFilePath = rpc(mim(), mongoose_system_metrics_file, location, []), 209: ReportLastModified = rpc(mim(), filelib, last_modified, [ReportFilePath]), 210: Fun = fun() -> 211: ReportLastModified < rpc(mim(), filelib, last_modified, [ReportFilePath]) 212: end, 213: mongoose_helper:wait_until(Fun, true), 214: %% now we read the content of the file and check if it's a valid JSON 215: {ok, File} = rpc(mim(), file, read_file, [ReportFilePath]), 216: jiffy:decode(File). 217: 218: module_opts_are_reported(_Config) -> 219: mongoose_helper:wait_until(fun are_modules_reported/0, true), 220: Backend = mongoose_helper:mnesia_or_rdbms_backend(), 221: check_module_backend(mod_bosh, mnesia), 222: check_module_backend(mod_event_pusher, push), 223: check_module_backend(mod_event_pusher_push, Backend), 224: check_module_backend(mod_http_upload, s3), 225: check_module_backend(mod_last, Backend), 226: check_module_backend(mod_muc, Backend), 227: check_module_backend(mod_muc_light, Backend), 228: check_module_backend(mod_offline, Backend), 229: check_module_backend(mod_privacy, Backend), 230: check_module_backend(mod_private, Backend), 231: check_module_backend(mod_pubsub, Backend), 232: check_module_opt(mod_push_service_mongoosepush, api_version, <<"\"v3\"">>), 233: check_module_backend(mod_roster, Backend), 234: check_module_backend(mod_vcard, Backend). 235: 236: rdbms_module_opts_are_reported(_Config) -> 237: mongoose_helper:wait_until(fun are_modules_reported/0, true), 238: check_module_backend(mod_auth_token, rdbms), 239: check_module_backend(mod_inbox, rdbms), 240: check_module_backend(mod_mam_meta, rdbms). 241: 242: check_module_backend(Module, Backend) -> 243: check_module_opt(Module, backend, atom_to_binary(Backend)). 244: 245: mongoose_version_is_reported(_Config) -> 246: mongoose_helper:wait_until(fun is_mongoose_version_reported/0, true). 247: 248: cluster_uptime_is_reported(_Config) -> 249: mongoose_helper:wait_until(fun is_cluster_uptime_reported/0, true). 250: 251: xmpp_components_are_reported(Config) -> 252: CompOpts = ?config(component1, Config), 253: {Component, Addr, _} = connect_component(CompOpts), 254: mongoose_helper:wait_until(fun are_xmpp_components_reported/0, true), 255: mongoose_helper:wait_until(fun more_than_one_component_is_reported/0, true), 256: disconnect_component(Component, Addr). 257: 258: api_are_reported(_Config) -> 259: mongoose_helper:wait_until(fun is_api_reported/0, true). 260: 261: transport_mechanisms_are_reported(_Config) -> 262: mongoose_helper:wait_until(fun are_transport_mechanisms_reported/0, true). 263: 264: outgoing_pools_are_reported(_Config) -> 265: mongoose_helper:wait_until(fun are_outgoing_pools_reported/0, true). 266: 267: xmpp_stanzas_counts_are_reported(Config) -> 268: escalus:story(Config, [{alice,1}, {bob,1}], fun(Alice, Bob) -> 269: mongoose_helper:wait_until(fun is_message_count_reported/0, true), 270: mongoose_helper:wait_until(fun is_iq_count_reported/0, true), 271: Sent = get_metric_value(<<"xmppMessageSent">>), 272: Received = get_metric_value(<<"xmppMessageReceived">>), 273: escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Hi">>)), 274: escalus:assert(is_chat_message, [<<"Hi">>], escalus:wait_for_stanza(Bob)), 275: F = fun() -> assert_message_count_is_incremented(Sent, Received) end, 276: mongoose_helper:wait_until(F, ok) 277: end). 278: 279: config_type_is_reported(_Config) -> 280: mongoose_helper:wait_until(fun is_config_type_reported/0, true). 281: 282: just_removed_from_config_logs_question(_Config) -> 283: disable_system_metrics(mim3()), 284: %% WHEN 285: Result = distributed_helper:rpc( 286: mim3(), service_mongoose_system_metrics, verify_if_configured, []), 287: %% THEN 288: ?assertEqual(ignore, Result). 289: 290: in_config_unmodified_logs_request_for_agreement(_Config) -> 291: %% WHEN 292: disable_system_metrics(mim()), 293: logger_ct_backend:capture(warning), 294: enable_system_metrics(mim()), 295: %% THEN 296: FilterFun = fun(_, Msg) -> 297: re:run(Msg, "MongooseIM docs", [global]) /= nomatch 298: end, 299: mongoose_helper:wait_until(fun() -> length(logger_ct_backend:recv(FilterFun)) end, 1), 300: %% CLEAN 301: logger_ct_backend:stop_capture(), 302: disable_system_metrics(mim()). 303: 304: in_config_with_explicit_no_report_goes_off_silently(_Config) -> 305: %% WHEN 306: logger_ct_backend:capture(warning), 307: start_system_metrics_service(mim(), #{report => false}), 308: logger_ct_backend:stop_capture(), 309: %% THEN 310: FilterFun = fun(warning, Msg) -> 311: re:run(Msg, "MongooseIM docs", [global]) /= nomatch; 312: (_,_) -> false 313: end, 314: [] = logger_ct_backend:recv(FilterFun), 315: %% CLEAN 316: disable_system_metrics(mim()). 317: 318: in_config_with_explicit_reporting_goes_on_silently(_Config) -> 319: %% WHEN 320: logger_ct_backend:capture(warning), 321: start_system_metrics_service(mim(), #{report => true}), 322: logger_ct_backend:stop_capture(), 323: %% THEN 324: FilterFun = fun(warning, Msg) -> 325: re:run(Msg, "MongooseIM docs", [global]) /= nomatch; 326: (_,_) -> false 327: end, 328: [] = logger_ct_backend:recv(FilterFun), 329: %% CLEAN 330: disable_system_metrics(mim()). 331: 332: %%-------------------------------------------------------------------- 333: %% Helpers 334: %%-------------------------------------------------------------------- 335: 336: required_modules(CaseName) -> 337: lists:filter(fun({Module, _Opts}) -> is_module_supported(Module) end, 338: modules_to_test(CaseName)). 339: 340: modules_to_test(module_opts_are_reported) -> 341: Backend = mongoose_helper:mnesia_or_rdbms_backend(), 342: [required_module(mod_bosh), 343: required_module(mod_event_pusher, 344: #{push => config([modules, mod_event_pusher, push], #{backend => Backend})}), 345: required_module(mod_http_upload, s3), 346: required_module(mod_last, Backend), 347: required_module(mod_muc, Backend), 348: required_module(mod_muc_light, Backend), 349: required_module(mod_offline, Backend), 350: required_module(mod_privacy, Backend), 351: required_module(mod_private, Backend), 352: required_module(mod_pubsub, Backend), 353: required_module(mod_push_service_mongoosepush), 354: required_module(mod_roster, Backend), 355: required_module(mod_vcard, Backend)]; 356: modules_to_test(rdbms_module_opts_are_reported) -> 357: [required_module(mod_auth_token), 358: required_module(mod_inbox), 359: required_module(mod_mam_meta)]. 360: 361: required_module(Module) -> 362: required_module(Module, #{}). 363: 364: required_module(Module, Backend) when is_atom(Backend) -> 365: {Module, mod_config(Module, #{backend => Backend})}; 366: required_module(Module, Opts) -> 367: {Module, mod_config(Module, Opts)}. 368: 369: check_module_opt(Module, Key, Value) -> 370: case is_module_supported(Module) of 371: true -> 372: ?assertEqual(true, is_module_opt_reported(Module, Key, Value)); 373: false -> 374: ct:log("Skipping unsupported module ~p", [Module]) 375: end. 376: 377: is_module_supported(Module) -> 378: is_host_type_static() orelse supports_dynamic_domains(Module). 379: 380: is_host_type_static() -> 381: rpc(mim(), mongoose_domain_core, is_static, [host_type()]). 382: 383: supports_dynamic_domains(Module) -> 384: rpc(mim(), gen_mod, does_module_support, [Module, dynamic_domains]). 385: 386: all_event_have_the_same_client_id() -> 387: Tab = ets:tab2list(?ETS_TABLE), 388: UniqueSortedTab = lists:usort([Cid || #event{cid = Cid} <- Tab]), 389: 1 = length(UniqueSortedTab). 390: 391: is_host_count_reported() -> 392: is_in_table(<<"hosts">>). 393: 394: are_modules_reported() -> 395: is_in_table(<<"module">>). 396: 397: is_in_table(EventCategory) -> 398: Tab = ets:tab2list(?ETS_TABLE), 399: lists:any( 400: fun(#event{ec = EC}) -> 401: verify_category(EC, EventCategory) 402: end, Tab). 403: 404: verify_category(EC, <<"module">>) -> 405: Result = re:run(EC, "^mod_.*"), 406: case Result of 407: {match, _Captured} -> true; 408: nomatch -> false 409: end; 410: verify_category(EC, EC) -> 411: true; 412: verify_category(_EC, _EventCategory) -> 413: false. 414: 415: get_events_collection_size() -> 416: ets:info(?ETS_TABLE, size). 417: 418: enable_system_metrics(Node) -> 419: enable_system_metrics(Node, #{initial_report => 100, periodic_report => 100}). 420: 421: enable_system_metrics_with_configurable_tracking_id(Node) -> 422: enable_system_metrics(Node, #{initial_report => 100, periodic_report => 100, 423: tracking_id => ?TRACKING_ID_EXTRA}). 424: 425: enable_system_metrics(Node, Opts) -> 426: UrlArgs = [google_analytics_url, ?SERVER_URL], 427: ok = mongoose_helper:successful_rpc(Node, mongoose_config, set_opt, UrlArgs), 428: start_system_metrics_service(Node, Opts). 429: 430: start_system_metrics_service(Node, ExtraOpts) -> 431: Opts = config([services, service_mongoose_system_metrics], ExtraOpts), 432: dynamic_services:ensure_started(Node, service_mongoose_system_metrics, Opts). 433: 434: disable_system_metrics(Node) -> 435: dynamic_services:ensure_stopped(Node, service_mongoose_system_metrics), 436: mongoose_helper:successful_rpc(Node, mongoose_config, unset_opt, [ google_analytics_url ]). 437: 438: delete_prev_client_id(Node) -> 439: mongoose_helper:successful_rpc(Node, mnesia, delete_table, [service_mongoose_system_metrics]). 440: 441: create_events_collection() -> 442: ets:new(?ETS_TABLE, [duplicate_bag, named_table, public]). 443: 444: clear_events_collection() -> 445: ets:delete_all_objects(?ETS_TABLE). 446: 447: system_metrics_service_is_enabled(Node) -> 448: Pid = distributed_helper:rpc(Node, erlang, whereis, [service_mongoose_system_metrics]), 449: erlang:is_pid(Pid). 450: 451: system_metrics_service_is_disabled(Node) -> 452: not system_metrics_service_is_enabled(Node). 453: 454: events_are_reported_to_primary_tracking_id() -> 455: events_are_reported_to_tracking_ids([primary_tracking_id()]). 456: 457: events_are_reported_to_both_tracking_ids() -> 458: events_are_reported_to_tracking_ids([primary_tracking_id(), ?TRACKING_ID_EXTRA]). 459: 460: primary_tracking_id() -> 461: case os:getenv("CI") of 462: "true" -> ?TRACKING_ID_CI; 463: _ -> ?TRACKING_ID 464: end. 465: 466: events_are_reported_to_tracking_ids(ConfiguredTrackingIds) -> 467: Tab = ets:tab2list(?ETS_TABLE), 468: ActualTrackingIds = lists:usort([Tid || #event{tid = Tid} <- Tab]), 469: ExpectedTrackingIds = lists:sort([list_to_binary(Tid) || Tid <- ConfiguredTrackingIds]), 470: ?assertEqual(ExpectedTrackingIds, ActualTrackingIds). 471: 472: is_feature_reported(EventCategory, EventAction) -> 473: length(match_events(EventCategory, EventAction)) > 0. 474: 475: is_feature_reported(EventCategory, EventAction, EventLabel) -> 476: length(match_events(EventCategory, EventAction, EventLabel)) > 0. 477: 478: is_module_backend_reported(Module, Backend) -> 479: is_feature_reported(atom_to_binary(Module), <<"backend">>, atom_to_binary(Backend)). 480: 481: is_module_opt_reported(Module, Key, Value) -> 482: is_feature_reported(atom_to_binary(Module), atom_to_binary(Key), Value). 483: 484: is_mongoose_version_reported() -> 485: is_feature_reported(<<"cluster">>, <<"mim_version">>). 486: 487: is_cluster_uptime_reported() -> 488: is_feature_reported(<<"cluster">>, <<"uptime">>). 489: 490: are_xmpp_components_reported() -> 491: is_feature_reported(<<"cluster">>, <<"number_of_components">>). 492: 493: is_config_type_reported() -> 494: IsToml = is_feature_reported(<<"cluster">>, <<"config_type">>, <<"toml">>), 495: IsCfg = is_feature_reported(<<"cluster">>, <<"config_type">>, <<"cfg">>), 496: IsToml orelse IsCfg. 497: 498: is_api_reported() -> 499: is_in_table(<<"http_api">>). 500: 501: are_transport_mechanisms_reported() -> 502: is_in_table(<<"transport_mechanism">>). 503: 504: are_outgoing_pools_reported() -> 505: is_in_table(<<"outgoing_pools">>). 506: 507: is_iq_count_reported() -> 508: is_in_table(<<"xmppIqSent">>). 509: 510: is_message_count_reported() -> 511: is_in_table(<<"xmppMessageSent">>) andalso is_in_table(<<"xmppMessageReceived">>). 512: 513: assert_message_count_is_incremented(Sent, Received) -> 514: assert_increment(<<"xmppMessageSent">>, Sent), 515: assert_increment(<<"xmppMessageReceived">>, Received). 516: 517: assert_increment(EventCategory, InitialValue) -> 518: Events = match_events(EventCategory, integer_to_binary(InitialValue + 1), <<$1>>), 519: ?assertMatch([_], Events). % expect exactly one event with an increment of 1 520: 521: get_metric_value(EventCategory) -> 522: [#event{ea = Value} | _] = match_events(EventCategory), 523: binary_to_integer(Value). 524: 525: more_than_one_component_is_reported() -> 526: Events = match_events(<<"cluster">>, <<"number_of_components">>), 527: lists:any(fun(#event{el = EL}) -> 528: binary_to_integer(EL) > 0 529: end, Events). 530: 531: match_events(EC) -> 532: ets:match_object(?ETS_TABLE, #event{ec = EC, _ = '_'}). 533: 534: match_events(EC, EA) -> 535: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, _ = '_'}). 536: 537: match_events(EC, EA, EL) -> 538: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, el = EL, _ = '_'}). 539: 540: %%-------------------------------------------------------------------- 541: %% Cowboy handlers 542: %%-------------------------------------------------------------------- 543: handler_init(Req0) -> 544: {ok, Body, Req} = cowboy_req:read_body(Req0), 545: StrEvents = string:split(Body, "\n", all), 546: lists:map( 547: fun(StrEvent) -> 548: Event = str_to_event(StrEvent), 549: %% TODO there is a race condition when table is not available 550: ets:insert(?ETS_TABLE, Event) 551: end, StrEvents), 552: Req1 = cowboy_req:reply(200, #{}, <<"">>, Req), 553: {ok, Req1, no_state}. 554: 555: str_to_event(Qs) -> 556: StrParams = string:split(Qs, "&", all), 557: Params = lists:map( 558: fun(StrParam) -> 559: [StrKey, StrVal] = string:split(StrParam, "="), 560: {binary_to_atom(StrKey, utf8), StrVal} 561: end, StrParams), 562: #event{ 563: cid = get_el(cid, Params), 564: tid = get_el(tid, Params), 565: ec = get_el(ec, Params), 566: ea = get_el(ea, Params), 567: el = get_el(el, Params), 568: ev = get_el(ev, Params) 569: }. 570: 571: get_el(Key, Proplist) -> 572: proplists:get_value(Key, Proplist, undef). 573: 574: get_components(Opts, Config) -> 575: Components = [component1, component2, vjud_component], 576: [ {C, Opts ++ spec(C, Config)} || C <- Components ] ++ Config.