1: -module(service_mongoose_system_metrics_SUITE). 2: 3: -include_lib("common_test/include/ct.hrl"). 4: -include_lib("stdlib/include/ms_transform.hrl"). 5: -include_lib("eunit/include/eunit.hrl"). 6: 7: -define(SERVER_URL, "http://localhost:8765"). 8: -define(ETS_TABLE, qs). 9: -define(TRACKING_ID, "UA-151671255-2"). 10: -define(TRACKING_ID_CI, "UA-151671255-1"). 11: -define(TRACKING_ID_EXTRA, "UA-EXTRA-TRACKING-ID"). 12: 13: -record(event, { 14: cid = "", 15: tid = "", 16: ec = "", 17: ea = "", 18: ev = "", 19: el = "" }). 20: 21: %% API 22: -export([ 23: all/0, 24: suite/0, 25: groups/0, 26: init_per_suite/1, 27: end_per_suite/1, 28: init_per_group/2, 29: end_per_group/2, 30: init_per_testcase/2, 31: end_per_testcase/2 32: ]). 33: 34: -export([ 35: system_metrics_are_not_reported_when_not_allowed/1, 36: periodic_report_available/1, 37: all_clustered_mongooses_report_the_same_client_id/1, 38: system_metrics_are_reported_to_google_analytics_when_mim_starts/1, 39: system_metrics_are_reported_to_configurable_google_analytics/1, 40: system_metrics_are_reported_to_a_json_file/1, 41: module_backend_is_reported/1, 42: mongoose_version_is_reported/1, 43: cluster_uptime_is_reported/1, 44: xmpp_components_are_reported/1, 45: api_are_reported/1, 46: transport_mechanisms_are_reported/1, 47: outgoing_pools_are_reported/1, 48: xmpp_stanzas_counts_are_reported/1, 49: config_type_is_reported/1 50: ]). 51: 52: -export([ 53: just_removed_from_config_logs_question/1, 54: in_config_unmodified_logs_request_for_agreement/1, 55: in_config_with_explicit_no_report_goes_off_silently/1, 56: in_config_with_explicit_reporting_goes_on_silently/1 57: ]). 58: 59: -import(distributed_helper, [mim/0, mim2/0, mim3/0, 60: require_rpc_nodes/1 61: ]). 62: 63: -import(component_helper, [connect_component/1, 64: disconnect_component/2, 65: spec/2, 66: common/1]). 67: 68: -import(domain_helper, [host_type/0]). 69: 70: suite() -> 71: require_rpc_nodes([mim]). 72: 73: all() -> 74: [ 75: system_metrics_are_not_reported_when_not_allowed, 76: periodic_report_available, 77: all_clustered_mongooses_report_the_same_client_id, 78: system_metrics_are_reported_to_google_analytics_when_mim_starts, 79: system_metrics_are_reported_to_configurable_google_analytics, 80: system_metrics_are_reported_to_a_json_file, 81: module_backend_is_reported, 82: mongoose_version_is_reported, 83: cluster_uptime_is_reported, 84: xmpp_components_are_reported, 85: api_are_reported, 86: transport_mechanisms_are_reported, 87: outgoing_pools_are_reported, 88: xmpp_stanzas_counts_are_reported, 89: config_type_is_reported, 90: {group, log_transparency} 91: ]. 92: 93: groups() -> 94: [ 95: {log_transparency, [], [ 96: just_removed_from_config_logs_question, 97: in_config_unmodified_logs_request_for_agreement, 98: in_config_with_explicit_no_report_goes_off_silently, 99: in_config_with_explicit_reporting_goes_on_silently 100: ]} 101: ]. 102: 103: -define(APPS, [inets, crypto, ssl, ranch, cowlib, cowboy]). 104: 105: %%-------------------------------------------------------------------- 106: %% Suite configuration 107: %%-------------------------------------------------------------------- 108: init_per_suite(Config) -> 109: case system_metrics_service_is_enabled(mim()) of 110: false -> 111: ct:fail("service_mongoose_system_metrics is not running"); 112: true -> 113: [ {ok, _} = application:ensure_all_started(App) || App <- ?APPS ], 114: http_helper:start(8765, "/[...]", fun handler_init/1), 115: Config1 = escalus:init_per_suite(Config), 116: Config2 = ejabberd_node_utils:init(Config1), 117: dynamic_modules:save_modules(host_type(), Config2) 118: end. 119: 120: end_per_suite(Config) -> 121: http_helper:stop(), 122: Args = [{initial_report, timer:seconds(20)}, {periodic_report, timer:minutes(5)}], 123: [start_system_metrics_module(Node, Args) || Node <- [mim(), mim2()]], 124: dynamic_modules:restore_modules(Config), 125: escalus:end_per_suite(Config). 126: 127: %%-------------------------------------------------------------------- 128: %% Init & teardown 129: %%-------------------------------------------------------------------- 130: init_per_group(log_transparency, Config) -> 131: logger_ct_backend:start(), 132: logger_ct_backend:capture(warning), 133: Config; 134: init_per_group(_GroupName, Config) -> 135: Config. 136: 137: end_per_group(log_transparency, Config) -> 138: logger_ct_backend:stop_capture(), 139: Config; 140: end_per_group(_GroupName, Config) -> 141: Config. 142: 143: init_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) -> 144: create_events_collection(), 145: disable_system_metrics(mim()), 146: delete_prev_client_id(mim()), 147: Config; 148: init_per_testcase(all_clustered_mongooses_report_the_same_client_id, Config) -> 149: create_events_collection(), 150: distributed_helper:add_node_to_cluster(mim2(), Config), 151: enable_system_metrics(mim()), 152: enable_system_metrics(mim2()), 153: Config; 154: init_per_testcase(system_metrics_are_reported_to_configurable_google_analytics, Config) -> 155: create_events_collection(), 156: enable_system_metrics_with_configurable_tracking_id(mim()), 157: Config; 158: init_per_testcase(xmpp_components_are_reported, Config) -> 159: create_events_collection(), 160: Config1 = get_components(common(Config), Config), 161: enable_system_metrics(mim()), 162: Config1; 163: init_per_testcase(module_backend_is_reported, Config) -> 164: create_events_collection(), 165: DefModVCardConfig = config_parser_helper:default_mod_config(mod_vcard), 166: dynamic_modules:ensure_modules(host_type(), [{mod_vcard, DefModVCardConfig}]), 167: enable_system_metrics(mim()), 168: Config; 169: init_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) -> 170: create_events_collection(), 171: enable_system_metrics(mim()), 172: Config1 = escalus:create_users(Config, escalus:get_users([alice, bob])), 173: escalus:init_per_testcase(CN, Config1); 174: init_per_testcase(_TestcaseName, Config) -> 175: create_events_collection(), 176: enable_system_metrics(mim()), 177: Config. 178: 179: end_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) -> 180: clear_events_collection(), 181: delete_prev_client_id(mim()), 182: Config; 183: end_per_testcase(all_clustered_mongooses_report_the_same_client_id , Config) -> 184: clear_events_collection(), 185: delete_prev_client_id(mim()), 186: Nodes = [mim(), mim2()], 187: [ begin delete_prev_client_id(Node), disable_system_metrics(Node) end || Node <- Nodes ], 188: distributed_helper:remove_node_from_cluster(mim2(), Config), 189: Config; 190: end_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) -> 191: clear_events_collection(), 192: disable_system_metrics(mim()), 193: escalus:delete_users(Config, escalus:get_users([alice, bob])), 194: escalus:end_per_testcase(CN, Config); 195: end_per_testcase(_TestcaseName, Config) -> 196: clear_events_collection(), 197: disable_system_metrics(mim()), 198: delete_prev_client_id(mim()), 199: Config. 200: 201: 202: %%-------------------------------------------------------------------- 203: %% Tests 204: %%-------------------------------------------------------------------- 205: system_metrics_are_not_reported_when_not_allowed(_Config) -> 206: true = system_metrics_service_is_disabled(mim()). 207: 208: periodic_report_available(_Config) -> 209: ReportsNumber = get_events_collection_size(), 210: mongoose_helper:wait_until( 211: fun() -> 212: NewReportsNumber = get_events_collection_size(), 213: NewReportsNumber > ReportsNumber + 1 214: end, 215: true). 216: 217: all_clustered_mongooses_report_the_same_client_id(_Config) -> 218: mongoose_helper:wait_until(fun hosts_count_is_reported/0, true), 219: all_event_have_the_same_client_id(). 220: 221: system_metrics_are_reported_to_google_analytics_when_mim_starts(_Config) -> 222: mongoose_helper:wait_until(fun hosts_count_is_reported/0, true), 223: mongoose_helper:wait_until(fun modules_are_reported/0, true), 224: events_are_reported_to_primary_tracking_id(), 225: all_event_have_the_same_client_id(). 226: 227: system_metrics_are_reported_to_configurable_google_analytics(_Config) -> 228: mongoose_helper:wait_until(fun hosts_count_is_reported/0, true), 229: mongoose_helper:wait_until(fun modules_are_reported/0, true), 230: events_are_reported_to_both_tracking_ids(), 231: all_event_have_the_same_client_id(). 232: 233: system_metrics_are_reported_to_a_json_file(_Config) -> 234: ReportFilePath = distributed_helper:rpc(mim(), mongoose_system_metrics_file, location, []), 235: ReportLastModified = distributed_helper:rpc(mim(), filelib, last_modified, [ReportFilePath]), 236: Fun = fun() -> 237: ReportLastModified < distributed_helper:rpc(mim(), filelib, last_modified, [ReportFilePath]) 238: end, 239: mongoose_helper:wait_until(Fun, true), 240: %% now we read the content of the file and check if it's a valid JSON 241: {ok, File} = distributed_helper:rpc(mim(), file, read_file, [ReportFilePath]), 242: jiffy:decode(File). 243: 244: module_backend_is_reported(_Config) -> 245: mongoose_helper:wait_until(fun modules_are_reported/0, true), 246: mongoose_helper:wait_until(fun mod_vcard_backend_is_reported/0, true). 247: 248: mongoose_version_is_reported(_Config) -> 249: mongoose_helper:wait_until(fun mongoose_version_is_reported/0, true). 250: 251: cluster_uptime_is_reported(_Config) -> 252: mongoose_helper:wait_until(fun cluster_uptime_is_reported/0, true). 253: 254: xmpp_components_are_reported(Config) -> 255: CompOpts = ?config(component1, Config), 256: {Component, Addr, _} = connect_component(CompOpts), 257: mongoose_helper:wait_until(fun xmpp_components_are_reported/0, true), 258: mongoose_helper:wait_until(fun more_than_one_component_is_reported/0, true), 259: disconnect_component(Component, Addr). 260: 261: api_are_reported(_Config) -> 262: mongoose_helper:wait_until(fun api_are_reported/0, true). 263: 264: transport_mechanisms_are_reported(_Config) -> 265: mongoose_helper:wait_until(fun transport_mechanisms_are_reported/0, true). 266: 267: outgoing_pools_are_reported(_Config) -> 268: mongoose_helper:wait_until(fun outgoing_pools_are_reported/0, true). 269: 270: xmpp_stanzas_counts_are_reported(Config) -> 271: escalus:story(Config, [{alice,1}, {bob,1}], fun(Alice, Bob) -> 272: mongoose_helper:wait_until(fun message_count_is_reported/0, true), 273: mongoose_helper:wait_until(fun iq_count_is_reported/0, true), 274: Sent = get_metric_value(<<"xmppMessageSent">>), 275: Received = get_metric_value(<<"xmppMessageReceived">>), 276: escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Hi">>)), 277: escalus:assert(is_chat_message, [<<"Hi">>], escalus:wait_for_stanza(Bob)), 278: F = fun() -> assert_message_count_is_incremented(Sent, Received) end, 279: mongoose_helper:wait_until(F, ok) 280: end). 281: 282: config_type_is_reported(_Config) -> 283: mongoose_helper:wait_until(fun config_type_is_reported/0, true). 284: 285: just_removed_from_config_logs_question(_Config) -> 286: disable_system_metrics(mim3()), 287: remove_service_from_config(service_mongoose_system_metrics), 288: %% WHEN 289: Result = distributed_helper:rpc( 290: mim3(), service_mongoose_system_metrics, verify_if_configured, []), 291: %% THEN 292: ?assertEqual(ignore, Result). 293: 294: in_config_unmodified_logs_request_for_agreement(_Config) -> 295: %% WHEN 296: disable_system_metrics(mim()), 297: logger_ct_backend:capture(warning), 298: enable_system_metrics(mim()), 299: %% THEN 300: FilterFun = fun(_, Msg) -> 301: re:run(Msg, "MongooseIM docs", [global]) /= nomatch 302: end, 303: mongoose_helper:wait_until(fun() -> length(logger_ct_backend:recv(FilterFun)) end, 1), 304: %% CLEAN 305: logger_ct_backend:stop_capture(), 306: disable_system_metrics(mim()). 307: 308: in_config_with_explicit_no_report_goes_off_silently(_Config) -> 309: %% WHEN 310: logger_ct_backend:capture(warning), 311: start_system_metrics_module(mim(), [no_report]), 312: logger_ct_backend:stop_capture(), 313: %% THEN 314: FilterFun = fun(warning, Msg) -> 315: re:run(Msg, "MongooseIM docs", [global]) /= nomatch; 316: (_,_) -> false 317: end, 318: [] = logger_ct_backend:recv(FilterFun), 319: %% CLEAN 320: disable_system_metrics(mim()). 321: 322: in_config_with_explicit_reporting_goes_on_silently(_Config) -> 323: %% WHEN 324: logger_ct_backend:capture(warning), 325: start_system_metrics_module(mim(), [report]), 326: logger_ct_backend:stop_capture(), 327: %% THEN 328: FilterFun = fun(warning, Msg) -> 329: re:run(Msg, "MongooseIM docs", [global]) /= nomatch; 330: (_,_) -> false 331: end, 332: [] = logger_ct_backend:recv(FilterFun), 333: %% CLEAN 334: disable_system_metrics(mim()). 335: 336: %%-------------------------------------------------------------------- 337: %% Helpers 338: %%-------------------------------------------------------------------- 339: 340: all_event_have_the_same_client_id() -> 341: Tab = ets:tab2list(?ETS_TABLE), 342: UniqueSortedTab = lists:usort([Cid || #event{cid = Cid} <- Tab]), 343: 1 = length(UniqueSortedTab). 344: 345: hosts_count_is_reported() -> 346: is_in_table(<<"hosts">>). 347: 348: modules_are_reported() -> 349: is_in_table(<<"module">>). 350: 351: is_in_table(EventCategory) -> 352: Tab = ets:tab2list(?ETS_TABLE), 353: lists:any( 354: fun(#event{ec = EC}) -> 355: verify_category(EC, EventCategory) 356: end, Tab). 357: 358: verify_category(EC, <<"module">>) -> 359: Result = re:run(EC, "^mod_.*"), 360: case Result of 361: {match, _Captured} -> true; 362: nomatch -> false 363: end; 364: verify_category(EC, EC) -> 365: true; 366: verify_category(_EC, _EventCategory) -> 367: false. 368: 369: get_events_collection_size() -> 370: ets:info(?ETS_TABLE, size). 371: 372: enable_system_metrics(Node) -> 373: enable_system_metrics(Node, [{initial_report, 100}, {periodic_report, 100}]). 374: 375: enable_system_metrics(Node, Timers) -> 376: UrlArgs = [google_analytics_url, ?SERVER_URL], 377: ok = mongoose_helper:successful_rpc(Node, mongoose_config, set_opt, UrlArgs), 378: start_system_metrics_module(Node, Timers). 379: 380: enable_system_metrics_with_configurable_tracking_id(Node) -> 381: enable_system_metrics(Node, [{initial_report, 100}, {periodic_report, 100}, {tracking_id, ?TRACKING_ID_EXTRA}]). 382: 383: start_system_metrics_module(Node, Args) -> 384: distributed_helper:rpc( 385: Node, mongoose_service, start_service, [service_mongoose_system_metrics, Args]). 386: 387: disable_system_metrics(Node) -> 388: distributed_helper:rpc(Node, mongoose_service, stop_service, [service_mongoose_system_metrics]), 389: mongoose_helper:successful_rpc(Node, mongoose_config, unset_opt, [ google_analytics_url ]). 390: 391: delete_prev_client_id(Node) -> 392: mongoose_helper:successful_rpc(Node, mnesia, delete_table, [service_mongoose_system_metrics]). 393: 394: create_events_collection() -> 395: ets:new(?ETS_TABLE, [duplicate_bag, named_table, public]). 396: 397: clear_events_collection() -> 398: ets:delete_all_objects(?ETS_TABLE). 399: 400: system_metrics_service_is_enabled(Node) -> 401: Pid = distributed_helper:rpc(Node, erlang, whereis, [service_mongoose_system_metrics]), 402: erlang:is_pid(Pid). 403: 404: system_metrics_service_is_disabled(Node) -> 405: not system_metrics_service_is_enabled(Node). 406: 407: remove_service_from_config(Service) -> 408: Services = distributed_helper:rpc(mim3(), mongoose_config, get_opt, [services]), 409: NewServices = proplists:delete(Service, Services), 410: distributed_helper:rpc(mim3(), mongoose_config, set_opt, [services, NewServices]). 411: 412: events_are_reported_to_primary_tracking_id() -> 413: events_are_reported_to_tracking_ids([primary_tracking_id()]). 414: 415: events_are_reported_to_both_tracking_ids() -> 416: events_are_reported_to_tracking_ids([primary_tracking_id(), ?TRACKING_ID_EXTRA]). 417: 418: primary_tracking_id() -> 419: case os:getenv("CI") of 420: "true" -> ?TRACKING_ID_CI; 421: _ -> ?TRACKING_ID 422: end. 423: 424: events_are_reported_to_tracking_ids(ConfiguredTrackingIds) -> 425: Tab = ets:tab2list(?ETS_TABLE), 426: ActualTrackingIds = lists:usort([Tid || #event{tid = Tid} <- Tab]), 427: ExpectedTrackingIds = lists:sort([list_to_binary(Tid) || Tid <- ConfiguredTrackingIds]), 428: ?assertEqual(ExpectedTrackingIds, ActualTrackingIds). 429: 430: feature_is_reported(EventCategory, EventAction) -> 431: length(match_events(EventCategory, EventAction)) > 0. 432: 433: feature_is_reported(EventCategory, EventAction, EventLabel) -> 434: length(match_events(EventCategory, EventAction, EventLabel)) > 0. 435: 436: mod_vcard_backend_is_reported() -> 437: feature_is_reported(<<"mod_vcard">>, <<"backend">>). 438: 439: mongoose_version_is_reported() -> 440: feature_is_reported(<<"cluster">>, <<"mim_version">>). 441: 442: cluster_uptime_is_reported() -> 443: feature_is_reported(<<"cluster">>, <<"uptime">>). 444: 445: xmpp_components_are_reported() -> 446: feature_is_reported(<<"cluster">>, <<"number_of_components">>). 447: 448: config_type_is_reported() -> 449: IsToml = feature_is_reported(<<"cluster">>, <<"config_type">>, <<"toml">>), 450: IsCfg = feature_is_reported(<<"cluster">>, <<"config_type">>, <<"cfg">>), 451: IsToml orelse IsCfg. 452: 453: api_are_reported() -> 454: is_in_table(<<"http_api">>). 455: 456: transport_mechanisms_are_reported() -> 457: is_in_table(<<"transport_mechanism">>). 458: 459: outgoing_pools_are_reported() -> 460: is_in_table(<<"outgoing_pools">>). 461: 462: iq_count_is_reported() -> 463: is_in_table(<<"xmppIqSent">>). 464: 465: message_count_is_reported() -> 466: is_in_table(<<"xmppMessageSent">>) andalso is_in_table(<<"xmppMessageReceived">>). 467: 468: assert_message_count_is_incremented(Sent, Received) -> 469: assert_increment(<<"xmppMessageSent">>, Sent), 470: assert_increment(<<"xmppMessageReceived">>, Received). 471: 472: assert_increment(EventCategory, InitialValue) -> 473: Events = match_events(EventCategory, integer_to_binary(InitialValue + 1), <<$1>>), 474: ?assertMatch([_], Events). % expect exactly one event with an increment of 1 475: 476: get_metric_value(EventCategory) -> 477: [#event{ea = Value} | _] = match_events(EventCategory), 478: binary_to_integer(Value). 479: 480: more_than_one_component_is_reported() -> 481: Events = match_events(<<"cluster">>, <<"number_of_components">>), 482: lists:any(fun(#event{el = EL}) -> 483: binary_to_integer(EL) > 0 484: end, Events). 485: 486: match_events(EC) -> 487: ets:match_object(?ETS_TABLE, #event{ec = EC, _ = '_'}). 488: 489: match_events(EC, EA) -> 490: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, _ = '_'}). 491: 492: match_events(EC, EA, EL) -> 493: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, el = EL, _ = '_'}). 494: 495: %%-------------------------------------------------------------------- 496: %% Cowboy handlers 497: %%-------------------------------------------------------------------- 498: handler_init(Req0) -> 499: {ok, Body, Req} = cowboy_req:read_body(Req0), 500: StrEvents = string:split(Body, "\n", all), 501: lists:map( 502: fun(StrEvent) -> 503: Event = str_to_event(StrEvent), 504: %% TODO there is a race condition when table is not available 505: ets:insert(?ETS_TABLE, Event) 506: end, StrEvents), 507: Req1 = cowboy_req:reply(200, #{}, <<"">>, Req), 508: {ok, Req1, no_state}. 509: 510: str_to_event(Qs) -> 511: StrParams = string:split(Qs, "&", all), 512: Params = lists:map( 513: fun(StrParam) -> 514: [StrKey, StrVal] = string:split(StrParam, "="), 515: {binary_to_atom(StrKey, utf8), StrVal} 516: end, StrParams), 517: #event{ 518: cid = get_el(cid, Params), 519: tid = get_el(tid, Params), 520: ec = get_el(ec, Params), 521: ea = get_el(ea, Params), 522: el = get_el(el, Params), 523: ev = get_el(ev, Params) 524: }. 525: 526: get_el(Key, Proplist) -> 527: proplists:get_value(Key, Proplist, undef). 528: 529: get_components(Opts, Config) -> 530: Components = [component1, component2, vjud_component], 531: [ {C, Opts ++ spec(C, Config)} || C <- Components ] ++ Config.