1: -module(service_mongoose_system_metrics_SUITE). 2: 3: -include_lib("common_test/include/ct.hrl"). 4: -include_lib("stdlib/include/ms_transform.hrl"). 5: -include_lib("eunit/include/eunit.hrl"). 6: 7: -define(SERVER_URL, "http://localhost:8765"). 8: -define(ETS_TABLE, qs). 9: -define(TRACKING_ID, "UA-151671255-2"). 10: -define(TRACKING_ID_CI, "UA-151671255-1"). 11: -define(TRACKING_ID_EXTRA, "UA-EXTRA-TRACKING-ID"). 12: 13: -record(event, { 14: cid = "", 15: tid = "", 16: ec = "", 17: ea = "", 18: ev = "", 19: el = "" }). 20: 21: %% API 22: -export([ 23: all/0, 24: suite/0, 25: groups/0, 26: init_per_suite/1, 27: end_per_suite/1, 28: init_per_group/2, 29: end_per_group/2, 30: init_per_testcase/2, 31: end_per_testcase/2 32: ]). 33: 34: -export([ 35: system_metrics_are_not_reported_when_not_allowed/1, 36: periodic_report_available/1, 37: all_clustered_mongooses_report_the_same_client_id/1, 38: system_metrics_are_reported_to_google_analytics_when_mim_starts/1, 39: system_metrics_are_reported_to_configurable_google_analytics/1, 40: system_metrics_are_reported_to_a_json_file/1, 41: module_backend_is_reported/1, 42: mongoose_version_is_reported/1, 43: cluster_uptime_is_reported/1, 44: xmpp_components_are_reported/1, 45: api_are_reported/1, 46: transport_mechanisms_are_reported/1, 47: outgoing_pools_are_reported/1, 48: xmpp_stanzas_counts_are_reported/1, 49: config_type_is_reported/1 50: ]). 51: 52: -export([ 53: just_removed_from_config_logs_question/1, 54: in_config_unmodified_logs_request_for_agreement/1, 55: in_config_with_explicit_no_report_goes_off_silently/1, 56: in_config_with_explicit_reporting_goes_on_silently/1 57: ]). 58: 59: -import(distributed_helper, [mim/0, mim2/0, mim3/0, 60: require_rpc_nodes/1 61: ]). 62: 63: -import(component_helper, [connect_component/1, 64: disconnect_component/2, 65: spec/2, 66: common/1]). 67: 68: -import(domain_helper, [host_type/0]). 69: 70: suite() -> 71: require_rpc_nodes([mim]). 72: 73: all() -> 74: [ 75: system_metrics_are_not_reported_when_not_allowed, 76: periodic_report_available, 77: all_clustered_mongooses_report_the_same_client_id, 78: system_metrics_are_reported_to_google_analytics_when_mim_starts, 79: system_metrics_are_reported_to_configurable_google_analytics, 80: system_metrics_are_reported_to_a_json_file, 81: module_backend_is_reported, 82: mongoose_version_is_reported, 83: cluster_uptime_is_reported, 84: xmpp_components_are_reported, 85: api_are_reported, 86: transport_mechanisms_are_reported, 87: outgoing_pools_are_reported, 88: xmpp_stanzas_counts_are_reported, 89: config_type_is_reported, 90: {group, log_transparency} 91: ]. 92: 93: groups() -> 94: [ 95: {log_transparency, [], [ 96: just_removed_from_config_logs_question, 97: in_config_unmodified_logs_request_for_agreement, 98: in_config_with_explicit_no_report_goes_off_silently, 99: in_config_with_explicit_reporting_goes_on_silently 100: ]} 101: ]. 102: 103: -define(APPS, [inets, crypto, ssl, ranch, cowlib, cowboy]). 104: 105: %%-------------------------------------------------------------------- 106: %% Suite configuration 107: %%-------------------------------------------------------------------- 108: init_per_suite(Config) -> 109: case system_metrics_service_is_enabled(mim()) of 110: false -> 111: ct:fail("service_mongoose_system_metrics is not running"); 112: true -> 113: [ {ok, _} = application:ensure_all_started(App) || App <- ?APPS ], 114: http_helper:start(8765, "/[...]", fun handler_init/1), 115: Config1 = escalus:init_per_suite(Config), 116: Config2 = ejabberd_node_utils:init(Config1), 117: dynamic_modules:save_modules(host_type(), Config2) 118: end. 119: 120: end_per_suite(Config) -> 121: http_helper:stop(), 122: Args = [{initial_report, timer:seconds(20)}, {periodic_report, timer:minutes(5)}], 123: [start_system_metrics_module(Node, Args) || Node <- [mim(), mim2()]], 124: dynamic_modules:restore_modules(Config), 125: escalus:end_per_suite(Config). 126: 127: %%-------------------------------------------------------------------- 128: %% Init & teardown 129: %%-------------------------------------------------------------------- 130: init_per_group(log_transparency, Config) -> 131: logger_ct_backend:start(), 132: logger_ct_backend:capture(warning), 133: Config; 134: init_per_group(_GroupName, Config) -> 135: Config. 136: 137: end_per_group(log_transparency, Config) -> 138: logger_ct_backend:stop_capture(), 139: Config; 140: end_per_group(_GroupName, Config) -> 141: Config. 142: 143: init_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) -> 144: create_events_collection(), 145: disable_system_metrics(mim()), 146: delete_prev_client_id(mim()), 147: Config; 148: init_per_testcase(all_clustered_mongooses_report_the_same_client_id, Config) -> 149: create_events_collection(), 150: distributed_helper:add_node_to_cluster(mim2(), Config), 151: enable_system_metrics(mim()), 152: enable_system_metrics(mim2()), 153: Config; 154: init_per_testcase(system_metrics_are_reported_to_configurable_google_analytics, Config) -> 155: create_events_collection(), 156: enable_system_metrics_with_configurable_tracking_id(mim()), 157: Config; 158: init_per_testcase(xmpp_components_are_reported, Config) -> 159: create_events_collection(), 160: Config1 = get_components(common(Config), Config), 161: enable_system_metrics(mim()), 162: Config1; 163: init_per_testcase(module_backend_is_reported, Config) -> 164: create_events_collection(), 165: dynamic_modules:ensure_modules(host_type(), [{mod_vcard, []}]), 166: enable_system_metrics(mim()), 167: Config; 168: init_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) -> 169: create_events_collection(), 170: enable_system_metrics(mim()), 171: Config1 = escalus:create_users(Config, escalus:get_users([alice, bob])), 172: escalus:init_per_testcase(CN, Config1); 173: init_per_testcase(_TestcaseName, Config) -> 174: create_events_collection(), 175: enable_system_metrics(mim()), 176: Config. 177: 178: end_per_testcase(system_metrics_are_not_reported_when_not_allowed, Config) -> 179: clear_events_collection(), 180: delete_prev_client_id(mim()), 181: Config; 182: end_per_testcase(all_clustered_mongooses_report_the_same_client_id , Config) -> 183: clear_events_collection(), 184: delete_prev_client_id(mim()), 185: Nodes = [mim(), mim2()], 186: [ begin delete_prev_client_id(Node), disable_system_metrics(Node) end || Node <- Nodes ], 187: distributed_helper:remove_node_from_cluster(mim2(), Config), 188: Config; 189: end_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) -> 190: clear_events_collection(), 191: disable_system_metrics(mim()), 192: escalus:delete_users(Config, escalus:get_users([alice, bob])), 193: escalus:end_per_testcase(CN, Config); 194: end_per_testcase(_TestcaseName, Config) -> 195: clear_events_collection(), 196: disable_system_metrics(mim()), 197: delete_prev_client_id(mim()), 198: Config. 199: 200: 201: %%-------------------------------------------------------------------- 202: %% Tests 203: %%-------------------------------------------------------------------- 204: system_metrics_are_not_reported_when_not_allowed(_Config) -> 205: true = system_metrics_service_is_disabled(mim()). 206: 207: periodic_report_available(_Config) -> 208: ReportsNumber = get_events_collection_size(), 209: mongoose_helper:wait_until( 210: fun() -> 211: NewReportsNumber = get_events_collection_size(), 212: NewReportsNumber > ReportsNumber + 1 213: end, 214: true). 215: 216: all_clustered_mongooses_report_the_same_client_id(_Config) -> 217: mongoose_helper:wait_until(fun hosts_count_is_reported/0, true), 218: all_event_have_the_same_client_id(). 219: 220: system_metrics_are_reported_to_google_analytics_when_mim_starts(_Config) -> 221: mongoose_helper:wait_until(fun hosts_count_is_reported/0, true), 222: mongoose_helper:wait_until(fun modules_are_reported/0, true), 223: events_are_reported_to_primary_tracking_id(), 224: all_event_have_the_same_client_id(). 225: 226: system_metrics_are_reported_to_configurable_google_analytics(_Config) -> 227: mongoose_helper:wait_until(fun hosts_count_is_reported/0, true), 228: mongoose_helper:wait_until(fun modules_are_reported/0, true), 229: events_are_reported_to_both_tracking_ids(), 230: all_event_have_the_same_client_id(). 231: 232: system_metrics_are_reported_to_a_json_file(_Config) -> 233: ReportFilePath = distributed_helper:rpc(mim(), mongoose_system_metrics_file, location, []), 234: ReportLastModified = distributed_helper:rpc(mim(), filelib, last_modified, [ReportFilePath]), 235: Fun = fun() -> 236: ReportLastModified < distributed_helper:rpc(mim(), filelib, last_modified, [ReportFilePath]) 237: end, 238: mongoose_helper:wait_until(Fun, true), 239: %% now we read the content of the file and check if it's a valid JSON 240: {ok, File} = distributed_helper:rpc(mim(), file, read_file, [ReportFilePath]), 241: jiffy:decode(File). 242: 243: module_backend_is_reported(_Config) -> 244: mongoose_helper:wait_until(fun modules_are_reported/0, true), 245: mongoose_helper:wait_until(fun mod_vcard_backend_is_reported/0, true). 246: 247: mongoose_version_is_reported(_Config) -> 248: mongoose_helper:wait_until(fun mongoose_version_is_reported/0, true). 249: 250: cluster_uptime_is_reported(_Config) -> 251: mongoose_helper:wait_until(fun cluster_uptime_is_reported/0, true). 252: 253: xmpp_components_are_reported(Config) -> 254: CompOpts = ?config(component1, Config), 255: {Component, Addr, _} = connect_component(CompOpts), 256: mongoose_helper:wait_until(fun xmpp_components_are_reported/0, true), 257: mongoose_helper:wait_until(fun more_than_one_component_is_reported/0, true), 258: disconnect_component(Component, Addr). 259: 260: api_are_reported(_Config) -> 261: mongoose_helper:wait_until(fun api_are_reported/0, true). 262: 263: transport_mechanisms_are_reported(_Config) -> 264: mongoose_helper:wait_until(fun transport_mechanisms_are_reported/0, true). 265: 266: outgoing_pools_are_reported(_Config) -> 267: mongoose_helper:wait_until(fun outgoing_pools_are_reported/0, true). 268: 269: xmpp_stanzas_counts_are_reported(Config) -> 270: escalus:story(Config, [{alice,1}, {bob,1}], fun(Alice, Bob) -> 271: mongoose_helper:wait_until(fun message_count_is_reported/0, true), 272: mongoose_helper:wait_until(fun iq_count_is_reported/0, true), 273: Sent = get_metric_value(<<"xmppMessageSent">>), 274: Received = get_metric_value(<<"xmppMessageReceived">>), 275: escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Hi">>)), 276: escalus:assert(is_chat_message, [<<"Hi">>], escalus:wait_for_stanza(Bob)), 277: F = fun() -> assert_message_count_is_incremented(Sent, Received) end, 278: mongoose_helper:wait_until(F, ok) 279: end). 280: 281: config_type_is_reported(_Config) -> 282: mongoose_helper:wait_until(fun config_type_is_reported/0, true). 283: 284: just_removed_from_config_logs_question(_Config) -> 285: disable_system_metrics(mim3()), 286: remove_service_from_config(service_mongoose_system_metrics), 287: %% WHEN 288: Result = distributed_helper:rpc( 289: mim3(), service_mongoose_system_metrics, verify_if_configured, []), 290: %% THEN 291: ?assertEqual(ignore, Result). 292: 293: in_config_unmodified_logs_request_for_agreement(_Config) -> 294: %% WHEN 295: disable_system_metrics(mim()), 296: logger_ct_backend:capture(warning), 297: enable_system_metrics(mim()), 298: %% THEN 299: FilterFun = fun(_, Msg) -> 300: re:run(Msg, "MongooseIM docs", [global]) /= nomatch 301: end, 302: mongoose_helper:wait_until(fun() -> length(logger_ct_backend:recv(FilterFun)) end, 1), 303: %% CLEAN 304: logger_ct_backend:stop_capture(), 305: disable_system_metrics(mim()). 306: 307: in_config_with_explicit_no_report_goes_off_silently(_Config) -> 308: %% WHEN 309: logger_ct_backend:capture(warning), 310: start_system_metrics_module(mim(), [no_report]), 311: logger_ct_backend:stop_capture(), 312: %% THEN 313: FilterFun = fun(warning, Msg) -> 314: re:run(Msg, "MongooseIM docs", [global]) /= nomatch; 315: (_,_) -> false 316: end, 317: [] = logger_ct_backend:recv(FilterFun), 318: %% CLEAN 319: disable_system_metrics(mim()). 320: 321: in_config_with_explicit_reporting_goes_on_silently(_Config) -> 322: %% WHEN 323: logger_ct_backend:capture(warning), 324: start_system_metrics_module(mim(), [report]), 325: logger_ct_backend:stop_capture(), 326: %% THEN 327: FilterFun = fun(warning, Msg) -> 328: re:run(Msg, "MongooseIM docs", [global]) /= nomatch; 329: (_,_) -> false 330: end, 331: [] = logger_ct_backend:recv(FilterFun), 332: %% CLEAN 333: disable_system_metrics(mim()). 334: 335: %%-------------------------------------------------------------------- 336: %% Helpers 337: %%-------------------------------------------------------------------- 338: 339: all_event_have_the_same_client_id() -> 340: Tab = ets:tab2list(?ETS_TABLE), 341: UniqueSortedTab = lists:usort([Cid || #event{cid = Cid} <- Tab]), 342: 1 = length(UniqueSortedTab). 343: 344: hosts_count_is_reported() -> 345: is_in_table(<<"hosts">>). 346: 347: modules_are_reported() -> 348: is_in_table(<<"module">>). 349: 350: is_in_table(EventCategory) -> 351: Tab = ets:tab2list(?ETS_TABLE), 352: lists:any( 353: fun(#event{ec = EC}) -> 354: verify_category(EC, EventCategory) 355: end, Tab). 356: 357: verify_category(EC, <<"module">>) -> 358: Result = re:run(EC, "^mod_.*"), 359: case Result of 360: {match, _Captured} -> true; 361: nomatch -> false 362: end; 363: verify_category(EC, EC) -> 364: true; 365: verify_category(_EC, _EventCategory) -> 366: false. 367: 368: get_events_collection_size() -> 369: ets:info(?ETS_TABLE, size). 370: 371: enable_system_metrics(Node) -> 372: enable_system_metrics(Node, [{initial_report, 100}, {periodic_report, 100}]). 373: 374: enable_system_metrics(Node, Timers) -> 375: UrlArgs = [google_analytics_url, ?SERVER_URL], 376: ok = mongoose_helper:successful_rpc(Node, mongoose_config, set_opt, UrlArgs), 377: start_system_metrics_module(Node, Timers). 378: 379: enable_system_metrics_with_configurable_tracking_id(Node) -> 380: enable_system_metrics(Node, [{initial_report, 100}, {periodic_report, 100}, {tracking_id, ?TRACKING_ID_EXTRA}]). 381: 382: start_system_metrics_module(Node, Args) -> 383: distributed_helper:rpc( 384: Node, mongoose_service, start_service, [service_mongoose_system_metrics, Args]). 385: 386: disable_system_metrics(Node) -> 387: distributed_helper:rpc(Node, mongoose_service, stop_service, [service_mongoose_system_metrics]), 388: mongoose_helper:successful_rpc(Node, mongoose_config, unset_opt, [ google_analytics_url ]). 389: 390: delete_prev_client_id(Node) -> 391: mongoose_helper:successful_rpc(Node, mnesia, delete_table, [service_mongoose_system_metrics]). 392: 393: create_events_collection() -> 394: ets:new(?ETS_TABLE, [duplicate_bag, named_table, public]). 395: 396: clear_events_collection() -> 397: ets:delete_all_objects(?ETS_TABLE). 398: 399: system_metrics_service_is_enabled(Node) -> 400: Pid = distributed_helper:rpc(Node, erlang, whereis, [service_mongoose_system_metrics]), 401: erlang:is_pid(Pid). 402: 403: system_metrics_service_is_disabled(Node) -> 404: not system_metrics_service_is_enabled(Node). 405: 406: remove_service_from_config(Service) -> 407: Services = distributed_helper:rpc(mim3(), mongoose_config, get_opt, [services]), 408: NewServices = proplists:delete(Service, Services), 409: distributed_helper:rpc(mim3(), mongoose_config, set_opt, [services, NewServices]). 410: 411: events_are_reported_to_primary_tracking_id() -> 412: events_are_reported_to_tracking_ids([primary_tracking_id()]). 413: 414: events_are_reported_to_both_tracking_ids() -> 415: events_are_reported_to_tracking_ids([primary_tracking_id(), ?TRACKING_ID_EXTRA]). 416: 417: primary_tracking_id() -> 418: case os:getenv("CI") of 419: "true" -> ?TRACKING_ID_CI; 420: _ -> ?TRACKING_ID 421: end. 422: 423: events_are_reported_to_tracking_ids(ConfiguredTrackingIds) -> 424: Tab = ets:tab2list(?ETS_TABLE), 425: ActualTrackingIds = lists:usort([Tid || #event{tid = Tid} <- Tab]), 426: ExpectedTrackingIds = lists:sort([list_to_binary(Tid) || Tid <- ConfiguredTrackingIds]), 427: ?assertEqual(ExpectedTrackingIds, ActualTrackingIds). 428: 429: feature_is_reported(EventCategory, EventAction) -> 430: length(match_events(EventCategory, EventAction)) > 0. 431: 432: feature_is_reported(EventCategory, EventAction, EventLabel) -> 433: length(match_events(EventCategory, EventAction, EventLabel)) > 0. 434: 435: mod_vcard_backend_is_reported() -> 436: feature_is_reported(<<"mod_vcard">>, <<"backend">>). 437: 438: mongoose_version_is_reported() -> 439: feature_is_reported(<<"cluster">>, <<"mim_version">>). 440: 441: cluster_uptime_is_reported() -> 442: feature_is_reported(<<"cluster">>, <<"uptime">>). 443: 444: xmpp_components_are_reported() -> 445: feature_is_reported(<<"cluster">>, <<"number_of_components">>). 446: 447: config_type_is_reported() -> 448: IsToml = feature_is_reported(<<"cluster">>, <<"config_type">>, <<"toml">>), 449: IsCfg = feature_is_reported(<<"cluster">>, <<"config_type">>, <<"cfg">>), 450: IsToml orelse IsCfg. 451: 452: api_are_reported() -> 453: is_in_table(<<"http_api">>). 454: 455: transport_mechanisms_are_reported() -> 456: is_in_table(<<"transport_mechanism">>). 457: 458: outgoing_pools_are_reported() -> 459: is_in_table(<<"outgoing_pools">>). 460: 461: iq_count_is_reported() -> 462: is_in_table(<<"xmppIqSent">>). 463: 464: message_count_is_reported() -> 465: is_in_table(<<"xmppMessageSent">>) andalso is_in_table(<<"xmppMessageReceived">>). 466: 467: assert_message_count_is_incremented(Sent, Received) -> 468: assert_increment(<<"xmppMessageSent">>, Sent), 469: assert_increment(<<"xmppMessageReceived">>, Received). 470: 471: assert_increment(EventCategory, InitialValue) -> 472: Events = match_events(EventCategory, integer_to_binary(InitialValue + 1), <<$1>>), 473: ?assertMatch([_], Events). % expect exactly one event with an increment of 1 474: 475: get_metric_value(EventCategory) -> 476: [#event{ea = Value} | _] = match_events(EventCategory), 477: binary_to_integer(Value). 478: 479: more_than_one_component_is_reported() -> 480: Events = match_events(<<"cluster">>, <<"number_of_components">>), 481: lists:any(fun(#event{el = EL}) -> 482: binary_to_integer(EL) > 0 483: end, Events). 484: 485: match_events(EC) -> 486: ets:match_object(?ETS_TABLE, #event{ec = EC, _ = '_'}). 487: 488: match_events(EC, EA) -> 489: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, _ = '_'}). 490: 491: match_events(EC, EA, EL) -> 492: ets:match_object(?ETS_TABLE, #event{ec = EC, ea = EA, el = EL, _ = '_'}). 493: 494: %%-------------------------------------------------------------------- 495: %% Cowboy handlers 496: %%-------------------------------------------------------------------- 497: handler_init(Req0) -> 498: {ok, Body, Req} = cowboy_req:read_body(Req0), 499: StrEvents = string:split(Body, "\n", all), 500: lists:map( 501: fun(StrEvent) -> 502: Event = str_to_event(StrEvent), 503: %% TODO there is a race condition when table is not available 504: ets:insert(?ETS_TABLE, Event) 505: end, StrEvents), 506: Req1 = cowboy_req:reply(200, #{}, <<"">>, Req), 507: {ok, Req1, no_state}. 508: 509: str_to_event(Qs) -> 510: StrParams = string:split(Qs, "&", all), 511: Params = lists:map( 512: fun(StrParam) -> 513: [StrKey, StrVal] = string:split(StrParam, "="), 514: {binary_to_atom(StrKey, utf8), StrVal} 515: end, StrParams), 516: #event{ 517: cid = get_el(cid, Params), 518: tid = get_el(tid, Params), 519: ec = get_el(ec, Params), 520: ea = get_el(ea, Params), 521: el = get_el(el, Params), 522: ev = get_el(ev, Params) 523: }. 524: 525: get_el(Key, Proplist) -> 526: proplists:get_value(Key, Proplist, undef). 527: 528: get_components(Opts, Config) -> 529: Components = [component1, component2, vjud_component], 530: [ {C, Opts ++ spec(C, Config)} || C <- Components ] ++ Config.