1: -module(mod_http_upload_SUITE). 2: 3: -include_lib("escalus/include/escalus.hrl"). 4: -include_lib("common_test/include/ct.hrl"). 5: -include_lib("eunit/include/eunit.hrl"). 6: -include_lib("escalus/include/escalus_xmlns.hrl"). 7: -include_lib("exml/include/exml.hrl"). 8: 9: -import(domain_helper, [host_type/0]). 10: 11: -define(NS_XDATA, <<"jabber:x:data">>). 12: -define(NS_HTTP_UPLOAD_030, <<"urn:xmpp:http:upload:0">>). 13: 14: -define(S3_HOSTNAME, "http://bucket.s3-eu-east-25.example.com"). 15: -define(S3_OPTS, ?MOD_HTTP_UPLOAD_OPTS(?S3_HOSTNAME, true)). 16: 17: -define(MINIO_HOSTNAME, "http://127.0.0.1:9000/mybucket/"). 18: -define(MINIO_OPTS(AddAcl), ?MOD_HTTP_UPLOAD_OPTS(?MINIO_HOSTNAME, AddAcl)). 19: 20: -define(MINIO_TEST_DATA, "qwerty"). 21: 22: -define(MOD_HTTP_UPLOAD_OPTS(Host, AddAcl), 23: [ 24: {max_file_size, 1234}, 25: {s3, [ 26: {bucket_url, Host}, 27: {add_acl, AddAcl}, 28: {region, "eu-east-25"}, 29: {access_key_id, "AKIAIAOAONIULXQGMOUA"}, 30: {secret_access_key, "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"} 31: ]} 32: ]). 33: 34: -export([all/0, groups/0, suite/0, 35: init_per_suite/1, end_per_suite/1, 36: init_per_group/2, end_per_group/2, 37: init_per_testcase/2, end_per_testcase/2]). 38: 39: -export([ 40: does_not_advertise_max_size_if_unset/1, 41: 42: test_minio_upload_without_content_type/1, 43: test_minio_upload_with_content_type/1, 44: 45: http_upload_item_discovery/1, 46: http_upload_feature_discovery/1, 47: advertises_max_file_size/1, 48: request_slot/1, 49: rejects_set_iq/1, 50: rejects_disco_set_iq/1, 51: rejects_feature_discovery_with_node/1, 52: get_url_ends_with_filename/1, 53: urls_contain_s3_hostname/1, 54: rejects_empty_filename/1, 55: rejects_negative_filesize/1, 56: rejects_invalid_size_type/1, 57: denies_slots_over_max_file_size/1, 58: sends_different_put_and_get_urls/1, 59: escapes_urls_once/1 60: ]). 61: 62: %%-------------------------------------------------------------------- 63: %% Suite configuration 64: %%-------------------------------------------------------------------- 65: 66: all() -> 67: [{group, mod_http_upload_s3}, {group, unset_size}, 68: {group, real_upload_with_acl}, {group, real_upload_without_acl}]. 69: 70: groups() -> 71: [{unset_size, [], [does_not_advertise_max_size_if_unset]}, 72: {real_upload_with_acl, [], [test_minio_upload_without_content_type, 73: test_minio_upload_with_content_type]}, 74: {real_upload_without_acl, [], [test_minio_upload_without_content_type, 75: test_minio_upload_with_content_type]}, 76: {mod_http_upload_s3, [], [ 77: http_upload_item_discovery, 78: http_upload_feature_discovery, 79: advertises_max_file_size, 80: request_slot, 81: rejects_set_iq, 82: rejects_disco_set_iq, 83: rejects_feature_discovery_with_node, 84: get_url_ends_with_filename, 85: urls_contain_s3_hostname, 86: rejects_empty_filename, 87: rejects_negative_filesize, 88: rejects_invalid_size_type, 89: denies_slots_over_max_file_size, 90: sends_different_put_and_get_urls, 91: escapes_urls_once 92: ]}]. 93: 94: suite() -> 95: escalus:suite(). 96: 97: %%-------------------------------------------------------------------- 98: %% Init & teardown 99: %%-------------------------------------------------------------------- 100: 101: init_per_suite(Config) -> 102: escalus:init_per_suite(Config). 103: 104: end_per_suite(Config) -> 105: escalus:end_per_suite(Config). 106: 107: init_per_group(unset_size, Config) -> 108: dynamic_modules:start(host_type(), mod_http_upload, [{max_file_size, undefined} | ?S3_OPTS]), 109: escalus:create_users(Config, escalus:get_users([bob])); 110: init_per_group(real_upload_without_acl, Config) -> 111: case mongoose_helper:should_minio_be_running(Config) of 112: true -> 113: dynamic_modules:start(host_type(), mod_http_upload, ?MINIO_OPTS(false)), 114: escalus:create_users(Config, escalus:get_users([bob])); 115: false -> {skip, "minio is not running"} 116: end; 117: init_per_group(real_upload_with_acl, Config) -> 118: case mongoose_helper:should_minio_be_running(Config) of 119: true -> 120: dynamic_modules:start(host_type(), mod_http_upload, ?MINIO_OPTS(true)), 121: [{with_acl, true} | escalus:create_users(Config, escalus:get_users([bob]))]; 122: false -> {skip, "minio is not running"} 123: end; 124: init_per_group(_, Config) -> 125: dynamic_modules:start(host_type(), mod_http_upload, ?S3_OPTS), 126: escalus:create_users(Config, escalus:get_users([bob])). 127: 128: end_per_group(_, Config) -> 129: dynamic_modules:stop(host_type(), mod_http_upload), 130: escalus:delete_users(Config, escalus:get_users([bob])). 131: 132: init_per_testcase(CaseName, Config) -> 133: escalus:init_per_testcase(CaseName, Config). 134: 135: end_per_testcase(CaseName, Config) -> 136: escalus:end_per_testcase(CaseName, Config). 137: 138: %%-------------------------------------------------------------------- 139: %% Service discovery test 140: %%-------------------------------------------------------------------- 141: 142: http_upload_item_discovery(Config) -> 143: escalus:story( 144: Config, [{bob, 1}], 145: fun(Bob) -> 146: ServJID = escalus_client:server(Bob), 147: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_items(ServJID)), 148: escalus:assert(is_iq_result, Result), 149: Query = exml_query:subelement(Result, <<"query">>), 150: Item = exml_query:subelement_with_attr(Query, <<"jid">>, upload_service(Bob)), 151: ?assertEqual(<<"HTTP File Upload">>, exml_query:attr(Item, <<"name">>)) 152: end). 153: 154: http_upload_feature_discovery(Config) -> 155: escalus:story( 156: Config, [{bob, 1}], 157: fun(Bob) -> 158: ServJID = escalus_client:server(Bob), 159: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_info(ServJID)), 160: escalus:assert(fun has_no_feature/2, [ns()], Result), 161: SubServJID = upload_service(Bob), 162: SubResult = escalus:send_and_wait(Bob, escalus_stanza:disco_info(SubServJID)), 163: escalus:assert(has_feature, [ns()], SubResult) 164: end). 165: 166: advertises_max_file_size(Config) -> 167: escalus:story( 168: Config, [{bob, 1}], 169: fun(Bob) -> 170: ServJID = upload_service(Bob), 171: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_info(ServJID)), 172: Forms = exml_query:paths(Result, [{element, <<"query">>}, {element, <<"x">>}]), 173: [Form] = lists:filter( 174: fun(F) -> has_field(<<"FORM_TYPE">>, <<"hidden">>, ns(), F) end, 175: Forms), 176: 177: escalus:assert(has_type, [<<"result">>], Form), 178: escalus:assert(has_ns, [?NS_XDATA], Form), 179: escalus:assert(fun has_field/4, [<<"max-file-size">>, undefined, <<"1234">>], Form), 180: escalus:assert(has_identity, [<<"store">>, <<"file">>], Result) 181: end). 182: 183: does_not_advertise_max_size_if_unset(Config) -> 184: escalus:story( 185: Config, [{bob, 1}], 186: fun(Bob) -> 187: ServJID = upload_service(Bob), 188: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_info(ServJID)), 189: undefined = exml_query:path(Result, {element, <<"x">>}), 190: escalus:assert(has_identity, [<<"store">>, <<"file">>], Result) 191: end). 192: 193: rejects_set_iq(Config) -> 194: escalus:story( 195: Config, [{bob, 1}], 196: fun(Bob) -> 197: ServJID = upload_service(Bob), 198: IQ = escalus_stanza:iq_set(ns(), []), 199: Request = escalus_stanza:to(IQ, ServJID), 200: Result = escalus:send_and_wait(Bob, Request), 201: escalus_assert:is_error(Result, <<"cancel">>, <<"not-allowed">>) 202: end). 203: 204: rejects_disco_set_iq(Config) -> 205: escalus:story( 206: Config, [{bob, 1}], 207: fun(Bob) -> 208: ServJID = upload_service(Bob), 209: IQ = escalus_stanza:iq_set(?NS_DISCO_INFO, []), 210: Request = escalus_stanza:to(IQ, ServJID), 211: Stanza = escalus:send_and_wait(Bob, Request), 212: escalus:assert(is_iq_error, [Request], Stanza), 213: escalus:assert(is_error, [<<"cancel">>, <<"not-allowed">>], Stanza), 214: escalus:assert(is_stanza_from, [ServJID], Stanza) 215: end). 216: 217: rejects_feature_discovery_with_node(Config) -> 218: escalus:story( 219: Config, [{bob, 1}], 220: fun(Bob) -> 221: ServJID = upload_service(Bob), 222: Request = escalus_stanza:disco_info(ServJID, <<"bad-node">>), 223: Stanza = escalus:send_and_wait(Bob, Request), 224: escalus:assert(is_iq_error, [Request], Stanza), 225: escalus:assert(is_error, [<<"cancel">>, <<"item-not-found">>], Stanza), 226: escalus:assert(is_stanza_from, [ServJID], Stanza) 227: end). 228: 229: request_slot(Config) -> 230: escalus:story( 231: Config, [{bob, 1}], 232: fun(Bob) -> 233: ServJID = upload_service(Bob), 234: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 235: Result = escalus:send_and_wait(Bob, Request), 236: escalus:assert(is_iq_result, Result), 237: escalus:assert(fun has_upload_namespace/1, Result), 238: escalus:assert(fun has_put_and_get_fields/1, Result) 239: end). 240: 241: get_url_ends_with_filename(Config) -> 242: escalus:story( 243: Config, [{bob, 1}], 244: fun(Bob) -> 245: ServJID = upload_service(Bob), 246: Filename = <<"filename.jpg">>, 247: Request = create_slot_request_stanza(ServJID, Filename, 123, undefined), 248: Result = escalus:send_and_wait(Bob, Request), 249: escalus:assert(fun path_ends_with/3, [<<"get">>, Filename], Result) 250: end). 251: 252: urls_contain_s3_hostname(Config) -> 253: escalus:story( 254: Config, [{bob, 1}], 255: fun(Bob) -> 256: ServJID = upload_service(Bob), 257: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 258: Result = escalus:send_and_wait(Bob, Request), 259: escalus:assert(fun url_contains/3, [<<"get">>, <<?S3_HOSTNAME>>], Result), 260: escalus:assert(fun url_contains/3, [<<"put">>, <<?S3_HOSTNAME>>], Result) 261: end). 262: 263: test_minio_upload_without_content_type(Config) -> 264: test_minio_upload(Config, undefined). 265: 266: test_minio_upload_with_content_type(Config) -> 267: test_minio_upload(Config, <<"text/plain">>). 268: 269: test_minio_upload(Config, ContentType) -> 270: escalus:story( 271: Config, [{bob, 1}], 272: fun(Bob) -> 273: ServJID = upload_service(Bob), 274: FileSize = length(?MINIO_TEST_DATA), 275: Request = create_slot_request_stanza(ServJID, <<"file.txt">>, FileSize, ContentType), 276: Result = escalus:send_and_wait(Bob, Request), 277: GetUrl = binary_to_list(extract_url(Result, <<"get">>)), 278: PutUrl = binary_to_list(extract_url(Result, <<"put">>)), 279: Header = generate_header(Config, ContentType), 280: PutRetValue = request(put, PutUrl, Header, ?MINIO_TEST_DATA), 281: ?assertMatch({200, _}, PutRetValue), 282: GetRetValue = request(get, GetUrl, [], []), 283: ?assertMatch({200, ?MINIO_TEST_DATA}, GetRetValue) 284: end). 285: 286: request(Method, PutUrl, Header, Data) -> 287: PURL = #{host := Host, port := Port, path := Path} = uri_string:parse(PutUrl), 288: {ok, ConnPid} = gun:open(Host, Port), 289: {ok, _} = gun:await_up(ConnPid), 290: FullPath = 291: case PURL of 292: #{query := Query} -> 293: Path ++ "?" ++ Query; 294: _ -> 295: Path 296: end, 297: StreamRef = 298: case Method of 299: put -> 300: gun:put(ConnPid, FullPath, Header, Data); 301: get -> 302: gun:get(ConnPid, FullPath) 303: end, 304: RespOpts = #{pid => ConnPid, stream_ref => StreamRef, acc => <<>>}, 305: #{status := Status, acc := Acc} = get_reponse(RespOpts), 306: ok = gun:close(ConnPid), 307: {Status, binary_to_list(Acc)}. 308: 309: get_reponse(#{pid := Pid, stream_ref := StreamRef, acc := Acc} = Opts) -> 310: case gun:await(Pid, StreamRef) of 311: {response, fin, Status, _} -> 312: Opts#{status => Status, acc => Acc}; 313: {response, nofin, Status, _} -> 314: get_reponse(Opts#{status => Status}); 315: {data, nofin, Data} -> 316: get_reponse(Opts#{acc => <<Acc/binary, Data/binary>>}); 317: {data, fin, Data} -> 318: Opts#{acc => <<Acc/binary, Data/binary>>}; 319: Error -> 320: Error 321: end. 322: 323: generate_header(Config, undefined) -> 324: case proplists:get_value(with_acl, Config, false) of 325: true -> 326: [{<<"x-amz-acl">>, <<"public-read">>}]; 327: false -> 328: [] 329: end; 330: generate_header(Config, ContentType) -> 331: [{<<"content-type">>, ContentType} | generate_header(Config, undefined)]. 332: 333: rejects_empty_filename(Config) -> 334: escalus:story( 335: Config, [{bob, 1}], 336: fun(Bob) -> 337: ServJID = upload_service(Bob), 338: Request = create_slot_request_stanza(ServJID, <<>>, 123, undefined), 339: Result = escalus:send_and_wait(Bob, Request), 340: escalus_assert:is_error(Result, <<"modify">>, <<"bad-request">>) 341: end). 342: 343: rejects_negative_filesize(Config) -> 344: escalus:story( 345: Config, [{bob, 1}], 346: fun(Bob) -> 347: ServJID = upload_service(Bob), 348: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, -1, undefined), 349: Result = escalus:send_and_wait(Bob, Request), 350: escalus_assert:is_error(Result, <<"modify">>, <<"bad-request">>) 351: end). 352: 353: rejects_invalid_size_type(Config) -> 354: escalus:story( 355: Config, [{bob, 1}], 356: fun(Bob) -> 357: ServJID = upload_service(Bob), 358: Request = create_slot_request_stanza(ServJID, <<"a.jpg">>, <<"filesize">>, undefined), 359: Result = escalus:send_and_wait(Bob, Request), 360: escalus_assert:is_error(Result, <<"modify">>, <<"bad-request">>) 361: end). 362: 363: denies_slots_over_max_file_size(Config) -> 364: escalus:story( 365: Config, [{bob, 1}], 366: fun(Bob) -> 367: ServJID = upload_service(Bob), 368: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 54321, undefined), 369: Result = escalus:send_and_wait(Bob, Request), 370: escalus:assert(is_error, [<<"modify">>, <<"not-acceptable">>], Result), 371: <<"1234">> = exml_query:path(Result, [{element, <<"error">>}, 372: {element, <<"file-too-large">>}, 373: {element, <<"max-file-size">>}, 374: cdata]) 375: end). 376: 377: sends_different_put_and_get_urls(Config) -> 378: escalus:story( 379: Config, [{bob, 1}], 380: fun(Bob) -> 381: ServJID = upload_service(Bob), 382: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 383: Result = escalus:send_and_wait(Bob, Request), 384: escalus:assert(fun urls_not_equal/1, Result) 385: end). 386: 387: escapes_urls_once(Config) -> 388: escalus:story( 389: Config, [{bob, 1}], 390: fun(Bob) -> 391: ServJID = upload_service(Bob), 392: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 393: Result = escalus:send_and_wait(Bob, Request), 394: escalus:assert(fun url_contains/3, [<<"put">>, <<"%3Bx-amz-acl">>], Result) 395: end). 396: 397: %%-------------------------------------------------------------------- 398: %% Test helpers 399: %%-------------------------------------------------------------------- 400: create_slot_request_stanza(Server, Filename, Size, ContentType) when is_integer(Size) -> 401: create_slot_request_stanza(Server, Filename, integer_to_binary(Size), ContentType); 402: create_slot_request_stanza(Server, Filename, BinSize, ContentType) -> 403: #xmlel{name = <<"iq">>, 404: attrs = [{<<"type">>, <<"get">>}, {<<"to">>, Server}], 405: children = [create_request_element(Filename, BinSize, ContentType)]}. 406: 407: create_request_element(Filename, BinSize, ContentType) -> 408: ContentTypeEl = case ContentType of 409: undefined -> []; 410: _ -> [{<<"content-type">>, ContentType}] 411: end, 412: #xmlel{name = <<"request">>, 413: attrs = [{<<"xmlns">>, ?NS_HTTP_UPLOAD_030}, 414: {<<"filename">>, Filename}, 415: {<<"size">>, BinSize} 416: | ContentTypeEl]}. 417: 418: has_upload_namespace(#xmlel{name = <<"iq">>, children = [#xmlel{ name = <<"slot">> } = Slot]}) -> 419: ?NS_HTTP_UPLOAD_030 == exml_query:attr(Slot, <<"xmlns">>); 420: has_upload_namespace(_) -> 421: false. 422: 423: has_no_feature(Feature, Stanza) -> 424: not escalus_pred:has_feature(Feature, Stanza). 425: 426: has_put_and_get_fields(Elem = #xmlel{name = <<"iq">>}) -> 427: PutUrl = extract_url(Elem, <<"put">>), 428: GetUrl = extract_url(Elem, <<"get">>), 429: is_binary(PutUrl) andalso is_binary(GetUrl) 430: andalso byte_size(PutUrl) > 0 andalso byte_size(GetUrl) > 0; 431: has_put_and_get_fields(_Elem) -> 432: false. 433: 434: path_ends_with(UrlType, Filename, Result) -> 435: Url = extract_url(Result, UrlType), 436: #{ path := Path } = uri_string:parse(Url), 437: FilenameSize = byte_size(Filename), 438: ReverseFilename = reverse(Filename), 439: case reverse(Path) of 440: <<ReverseFilename:FilenameSize/binary, _/binary>> -> true; 441: _ -> false 442: end. 443: 444: url_contains(UrlType, Filename, Result) -> 445: Url = extract_url(Result, UrlType), 446: binary:match(Url, Filename) =/= nomatch. 447: 448: urls_not_equal(Result) -> 449: Get = extract_url(Result, <<"get">>), 450: Put = extract_url(Result, <<"put">>), 451: Get =/= Put. 452: 453: reverse(List) when is_list(List) -> 454: list_to_binary(lists:reverse(List)); 455: reverse(Binary) -> 456: reverse(binary_to_list(Binary)). 457: 458: upload_service(Client) -> 459: <<"upload.", (escalus_client:server(Client))/binary>>. 460: 461: has_field(Var, Type, Value, Form) -> 462: Fields = Form#xmlel.children, 463: VarFits = fun(I) -> Var =:= undefined orelse exml_query:attr(I, <<"var">>) =:= Var end, 464: TypeFits = fun(I) -> Type =:= undefined orelse exml_query:attr(I, <<"type">>) =:= Type end, 465: ValueFits = 466: fun(I) -> 467: Value =:= undefined orelse 468: Value =:= exml_query:path(I, [{element, <<"value">>}, cdata]) 469: end, 470: lists:any(fun(Item) -> VarFits(Item) andalso TypeFits(Item) andalso ValueFits(Item) end, 471: Fields). 472: 473: extract_url(Result, UrlType) -> 474: exml_query:path(Result, [{element, <<"slot">>}, {element, UrlType}, {attr, <<"url">>}]). 475: 476: ns() -> ?NS_HTTP_UPLOAD_030.