1: -module(mod_http_upload_SUITE). 2: 3: -include_lib("escalus/include/escalus.hrl"). 4: -include_lib("common_test/include/ct.hrl"). 5: -include_lib("eunit/include/eunit.hrl"). 6: -include_lib("escalus/include/escalus_xmlns.hrl"). 7: -include_lib("exml/include/exml.hrl"). 8: 9: -import(domain_helper, [host_type/0]). 10: 11: -define(NS_XDATA, <<"jabber:x:data">>). 12: -define(NS_HTTP_UPLOAD_030, <<"urn:xmpp:http:upload:0">>). 13: 14: -define(S3_HOSTNAME, <<"http://bucket.s3-eu-east-25.example.com">>). 15: -define(MINIO_HOSTNAME, <<"http://127.0.0.1:9000/mybucket/">>). 16: 17: -define(MINIO_TEST_DATA, "qwerty"). 18: 19: -export([all/0, groups/0, suite/0, 20: init_per_suite/1, end_per_suite/1, 21: init_per_group/2, end_per_group/2, 22: init_per_testcase/2, end_per_testcase/2]). 23: 24: -export([ 25: test_minio_upload_without_content_type/1, 26: test_minio_upload_with_content_type/1, 27: http_upload_item_discovery/1, 28: http_upload_feature_discovery/1, 29: advertises_max_file_size/1, 30: request_slot/1, 31: rejects_set_iq/1, 32: rejects_disco_set_iq/1, 33: rejects_feature_discovery_with_node/1, 34: get_url_ends_with_filename/1, 35: get_url_ends_with_filename_with_unicode_characters/1, 36: urls_contain_s3_hostname/1, 37: rejects_empty_filename/1, 38: rejects_negative_filesize/1, 39: rejects_invalid_size_type/1, 40: denies_slots_over_max_file_size/1, 41: sends_different_put_and_get_urls/1, 42: escapes_urls_once/1 43: ]). 44: 45: %%-------------------------------------------------------------------- 46: %% Suite configuration 47: %%-------------------------------------------------------------------- 48: 49: all() -> 50: [{group, mod_http_upload_s3}, {group, real_upload_with_acl}, 51: {group, real_upload_without_acl}]. 52: 53: groups() -> 54: [{real_upload_with_acl, [], [test_minio_upload_without_content_type, 55: test_minio_upload_with_content_type]}, 56: {real_upload_without_acl, [], [test_minio_upload_without_content_type, 57: test_minio_upload_with_content_type]}, 58: {mod_http_upload_s3, [], [ 59: http_upload_item_discovery, 60: http_upload_feature_discovery, 61: advertises_max_file_size, 62: request_slot, 63: rejects_set_iq, 64: rejects_disco_set_iq, 65: rejects_feature_discovery_with_node, 66: get_url_ends_with_filename, 67: get_url_ends_with_filename_with_unicode_characters, 68: urls_contain_s3_hostname, 69: rejects_empty_filename, 70: rejects_negative_filesize, 71: rejects_invalid_size_type, 72: denies_slots_over_max_file_size, 73: sends_different_put_and_get_urls, 74: escapes_urls_once 75: ]}]. 76: 77: suite() -> 78: escalus:suite(). 79: 80: %%-------------------------------------------------------------------- 81: %% Init & teardown 82: %%-------------------------------------------------------------------- 83: 84: init_per_suite(Config) -> 85: escalus:init_per_suite(Config). 86: end_per_suite(Config) -> 87: escalus:end_per_suite(Config). 88: 89: init_per_group(real_upload_without_acl, Config) -> 90: case mongoose_helper:should_minio_be_running(Config) of 91: true -> 92: dynamic_modules:start(host_type(), mod_http_upload, 93: create_opts(?MINIO_HOSTNAME, false)), 94: escalus:create_users(Config, escalus:get_users([bob])); 95: false -> {skip, "minio is not running"} 96: end; 97: init_per_group(real_upload_with_acl, Config) -> 98: case mongoose_helper:should_minio_be_running(Config) of 99: true -> 100: dynamic_modules:start(host_type(), mod_http_upload, create_opts(?MINIO_HOSTNAME, true)), 101: [{with_acl, true} | escalus:create_users(Config, escalus:get_users([bob]))]; 102: false -> {skip, "minio is not running"} 103: end; 104: init_per_group(_, Config) -> 105: dynamic_modules:start(host_type(), mod_http_upload, create_opts(?S3_HOSTNAME, true)), 106: escalus:create_users(Config, escalus:get_users([bob])). 107: 108: end_per_group(_, Config) -> 109: dynamic_modules:stop(host_type(), mod_http_upload), 110: escalus:delete_users(Config, escalus:get_users([bob])). 111: 112: init_per_testcase(CaseName, Config) -> 113: escalus:init_per_testcase(CaseName, Config). 114: 115: end_per_testcase(CaseName, Config) -> 116: escalus:end_per_testcase(CaseName, Config). 117: 118: create_opts(Host, AddAcl) -> 119: config_parser_helper:mod_config(mod_http_upload, 120: #{ 121: max_file_size => 1234, 122: s3 => #{ 123: bucket_url => Host, 124: add_acl => AddAcl, 125: region => <<"eu-east-25">>, 126: access_key_id => <<"AKIAIAOAONIULXQGMOUA">>, 127: secret_access_key => <<"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">> 128: } 129: }). 130: 131: %%-------------------------------------------------------------------- 132: %% Service discovery test 133: %%-------------------------------------------------------------------- 134: 135: http_upload_item_discovery(Config) -> 136: escalus:story( 137: Config, [{bob, 1}], 138: fun(Bob) -> 139: ServJID = escalus_client:server(Bob), 140: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_items(ServJID)), 141: escalus:assert(is_iq_result, Result), 142: Query = exml_query:subelement(Result, <<"query">>), 143: Item = exml_query:subelement_with_attr(Query, <<"jid">>, upload_service(Bob)), 144: ?assertEqual(<<"HTTP File Upload">>, exml_query:attr(Item, <<"name">>)) 145: end). 146: 147: http_upload_feature_discovery(Config) -> 148: escalus:story( 149: Config, [{bob, 1}], 150: fun(Bob) -> 151: ServJID = escalus_client:server(Bob), 152: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_info(ServJID)), 153: escalus:assert(fun has_no_feature/2, [ns()], Result), 154: SubServJID = upload_service(Bob), 155: SubResult = escalus:send_and_wait(Bob, escalus_stanza:disco_info(SubServJID)), 156: escalus:assert(has_feature, [ns()], SubResult) 157: end). 158: 159: advertises_max_file_size(Config) -> 160: escalus:story( 161: Config, [{bob, 1}], 162: fun(Bob) -> 163: ServJID = upload_service(Bob), 164: Result = escalus:send_and_wait(Bob, escalus_stanza:disco_info(ServJID)), 165: Forms = exml_query:paths(Result, [{element, <<"query">>}, {element, <<"x">>}]), 166: [Form] = lists:filter( 167: fun(F) -> has_field(<<"FORM_TYPE">>, <<"hidden">>, ns(), F) end, 168: Forms), 169: 170: escalus:assert(has_type, [<<"result">>], Form), 171: escalus:assert(has_ns, [?NS_XDATA], Form), 172: escalus:assert(fun has_field/4, [<<"max-file-size">>, undefined, <<"1234">>], Form), 173: escalus:assert(has_identity, [<<"store">>, <<"file">>], Result) 174: end). 175: 176: rejects_set_iq(Config) -> 177: escalus:story( 178: Config, [{bob, 1}], 179: fun(Bob) -> 180: ServJID = upload_service(Bob), 181: IQ = escalus_stanza:iq_set(ns(), []), 182: Request = escalus_stanza:to(IQ, ServJID), 183: Result = escalus:send_and_wait(Bob, Request), 184: escalus_assert:is_error(Result, <<"cancel">>, <<"not-allowed">>) 185: end). 186: 187: rejects_disco_set_iq(Config) -> 188: escalus:story( 189: Config, [{bob, 1}], 190: fun(Bob) -> 191: ServJID = upload_service(Bob), 192: IQ = escalus_stanza:iq_set(?NS_DISCO_INFO, []), 193: Request = escalus_stanza:to(IQ, ServJID), 194: Stanza = escalus:send_and_wait(Bob, Request), 195: escalus:assert(is_iq_error, [Request], Stanza), 196: escalus:assert(is_error, [<<"cancel">>, <<"not-allowed">>], Stanza), 197: escalus:assert(is_stanza_from, [ServJID], Stanza) 198: end). 199: 200: rejects_feature_discovery_with_node(Config) -> 201: escalus:story( 202: Config, [{bob, 1}], 203: fun(Bob) -> 204: ServJID = upload_service(Bob), 205: Request = escalus_stanza:disco_info(ServJID, <<"bad-node">>), 206: Stanza = escalus:send_and_wait(Bob, Request), 207: escalus:assert(is_iq_error, [Request], Stanza), 208: escalus:assert(is_error, [<<"cancel">>, <<"item-not-found">>], Stanza), 209: escalus:assert(is_stanza_from, [ServJID], Stanza) 210: end). 211: 212: request_slot(Config) -> 213: escalus:story( 214: Config, [{bob, 1}], 215: fun(Bob) -> 216: ServJID = upload_service(Bob), 217: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 218: Result = escalus:send_and_wait(Bob, Request), 219: escalus:assert(is_iq_result, Result), 220: escalus:assert(fun has_upload_namespace/1, Result), 221: escalus:assert(fun has_put_and_get_fields/1, Result) 222: end). 223: 224: get_url_ends_with_filename(Config) -> 225: escalus:story( 226: Config, [{bob, 1}], 227: fun(Bob) -> 228: ServJID = upload_service(Bob), 229: Filename = <<"filename.jpg">>, 230: Request = create_slot_request_stanza(ServJID, Filename, 123, undefined), 231: Result = escalus:send_and_wait(Bob, Request), 232: escalus:assert(fun path_ends_with/3, [<<"get">>, Filename], Result) 233: end). 234: 235: get_url_ends_with_filename_with_unicode_characters(Config) -> 236: escalus:story( 237: Config, [{bob, 1}], 238: fun(Bob) -> 239: ServJID = upload_service(Bob), 240: Filename = unicode:characters_to_binary("très cool.jpg"), 241: Request = create_slot_request_stanza(ServJID, Filename, 123, undefined), 242: Result = escalus:send_and_wait(Bob, Request), 243: escalus:assert(fun path_ends_with/3, [<<"get">>, <<"tr%C3%A8s%20cool.jpg">>], Result) 244: end). 245: 246: urls_contain_s3_hostname(Config) -> 247: escalus:story( 248: Config, [{bob, 1}], 249: fun(Bob) -> 250: ServJID = upload_service(Bob), 251: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 252: Result = escalus:send_and_wait(Bob, Request), 253: escalus:assert(fun url_contains/3, [<<"get">>, ?S3_HOSTNAME], Result), 254: escalus:assert(fun url_contains/3, [<<"put">>, ?S3_HOSTNAME], Result) 255: end). 256: 257: test_minio_upload_without_content_type(Config) -> 258: test_minio_upload(Config, undefined). 259: 260: test_minio_upload_with_content_type(Config) -> 261: test_minio_upload(Config, <<"text/plain">>). 262: 263: test_minio_upload(Config, ContentType) -> 264: escalus:story( 265: Config, [{bob, 1}], 266: fun(Bob) -> 267: ServJID = upload_service(Bob), 268: FileSize = length(?MINIO_TEST_DATA), 269: Request = create_slot_request_stanza(ServJID, <<"file.txt">>, FileSize, ContentType), 270: Result = escalus:send_and_wait(Bob, Request), 271: GetUrl = binary_to_list(extract_url(Result, <<"get">>)), 272: PutUrl = binary_to_list(extract_url(Result, <<"put">>)), 273: Header = generate_header(Config, ContentType), 274: PutRetValue = request(put, PutUrl, Header, ?MINIO_TEST_DATA), 275: ?assertMatch({200, _}, PutRetValue), 276: GetRetValue = request(get, GetUrl, [], []), 277: ?assertMatch({200, ?MINIO_TEST_DATA}, GetRetValue) 278: end). 279: 280: request(Method, PutUrl, Header, Data) -> 281: PURL = #{host := Host, port := Port, path := Path} = uri_string:parse(PutUrl), 282: {ok, ConnPid} = gun:open(Host, Port), 283: {ok, _} = gun:await_up(ConnPid), 284: FullPath = 285: case PURL of 286: #{query := Query} -> 287: Path ++ "?" ++ Query; 288: _ -> 289: Path 290: end, 291: StreamRef = 292: case Method of 293: put -> 294: gun:put(ConnPid, FullPath, Header, Data); 295: get -> 296: gun:get(ConnPid, FullPath) 297: end, 298: RespOpts = #{pid => ConnPid, stream_ref => StreamRef, acc => <<>>}, 299: #{status := Status, acc := Acc} = get_reponse(RespOpts), 300: ok = gun:close(ConnPid), 301: {Status, binary_to_list(Acc)}. 302: 303: get_reponse(#{pid := Pid, stream_ref := StreamRef, acc := Acc} = Opts) -> 304: case gun:await(Pid, StreamRef) of 305: {response, fin, Status, _} -> 306: Opts#{status => Status, acc => Acc}; 307: {response, nofin, Status, _} -> 308: get_reponse(Opts#{status => Status}); 309: {data, nofin, Data} -> 310: get_reponse(Opts#{acc => <<Acc/binary, Data/binary>>}); 311: {data, fin, Data} -> 312: Opts#{acc => <<Acc/binary, Data/binary>>}; 313: Error -> 314: Error 315: end. 316: 317: generate_header(Config, undefined) -> 318: case proplists:get_value(with_acl, Config, false) of 319: true -> 320: [{<<"x-amz-acl">>, <<"public-read">>}]; 321: false -> 322: [] 323: end; 324: generate_header(Config, ContentType) -> 325: [{<<"content-type">>, ContentType} | generate_header(Config, undefined)]. 326: 327: rejects_empty_filename(Config) -> 328: escalus:story( 329: Config, [{bob, 1}], 330: fun(Bob) -> 331: ServJID = upload_service(Bob), 332: Request = create_slot_request_stanza(ServJID, <<>>, 123, undefined), 333: Result = escalus:send_and_wait(Bob, Request), 334: escalus_assert:is_error(Result, <<"modify">>, <<"bad-request">>) 335: end). 336: 337: rejects_negative_filesize(Config) -> 338: escalus:story( 339: Config, [{bob, 1}], 340: fun(Bob) -> 341: ServJID = upload_service(Bob), 342: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, -1, undefined), 343: Result = escalus:send_and_wait(Bob, Request), 344: escalus_assert:is_error(Result, <<"modify">>, <<"bad-request">>) 345: end). 346: 347: rejects_invalid_size_type(Config) -> 348: escalus:story( 349: Config, [{bob, 1}], 350: fun(Bob) -> 351: ServJID = upload_service(Bob), 352: Request = create_slot_request_stanza(ServJID, <<"a.jpg">>, <<"filesize">>, undefined), 353: Result = escalus:send_and_wait(Bob, Request), 354: escalus_assert:is_error(Result, <<"modify">>, <<"bad-request">>) 355: end). 356: 357: denies_slots_over_max_file_size(Config) -> 358: escalus:story( 359: Config, [{bob, 1}], 360: fun(Bob) -> 361: ServJID = upload_service(Bob), 362: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 54321, undefined), 363: Result = escalus:send_and_wait(Bob, Request), 364: escalus:assert(is_error, [<<"modify">>, <<"not-acceptable">>], Result), 365: <<"1234">> = exml_query:path(Result, [{element, <<"error">>}, 366: {element, <<"file-too-large">>}, 367: {element, <<"max-file-size">>}, 368: cdata]) 369: end). 370: 371: sends_different_put_and_get_urls(Config) -> 372: escalus:story( 373: Config, [{bob, 1}], 374: fun(Bob) -> 375: ServJID = upload_service(Bob), 376: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 377: Result = escalus:send_and_wait(Bob, Request), 378: escalus:assert(fun urls_not_equal/1, Result) 379: end). 380: 381: escapes_urls_once(Config) -> 382: escalus:story( 383: Config, [{bob, 1}], 384: fun(Bob) -> 385: ServJID = upload_service(Bob), 386: Request = create_slot_request_stanza(ServJID, <<"filename.jpg">>, 123, undefined), 387: Result = escalus:send_and_wait(Bob, Request), 388: escalus:assert(fun url_contains/3, [<<"put">>, <<"%3Bx-amz-acl">>], Result) 389: end). 390: 391: %%-------------------------------------------------------------------- 392: %% Test helpers 393: %%-------------------------------------------------------------------- 394: create_slot_request_stanza(Server, Filename, Size, ContentType) when is_integer(Size) -> 395: create_slot_request_stanza(Server, Filename, integer_to_binary(Size), ContentType); 396: create_slot_request_stanza(Server, Filename, BinSize, ContentType) -> 397: #xmlel{name = <<"iq">>, 398: attrs = [{<<"type">>, <<"get">>}, {<<"to">>, Server}], 399: children = [create_request_element(Filename, BinSize, ContentType)]}. 400: 401: create_request_element(Filename, BinSize, ContentType) -> 402: ContentTypeEl = case ContentType of 403: undefined -> []; 404: _ -> [{<<"content-type">>, ContentType}] 405: end, 406: #xmlel{name = <<"request">>, 407: attrs = [{<<"xmlns">>, ?NS_HTTP_UPLOAD_030}, 408: {<<"filename">>, Filename}, 409: {<<"size">>, BinSize} 410: | ContentTypeEl]}. 411: 412: has_upload_namespace(#xmlel{name = <<"iq">>, children = [#xmlel{ name = <<"slot">> } = Slot]}) -> 413: ?NS_HTTP_UPLOAD_030 == exml_query:attr(Slot, <<"xmlns">>); 414: has_upload_namespace(_) -> 415: false. 416: 417: has_no_feature(Feature, Stanza) -> 418: not escalus_pred:has_feature(Feature, Stanza). 419: 420: has_put_and_get_fields(Elem = #xmlel{name = <<"iq">>}) -> 421: PutUrl = extract_url(Elem, <<"put">>), 422: GetUrl = extract_url(Elem, <<"get">>), 423: is_binary(PutUrl) andalso is_binary(GetUrl) 424: andalso byte_size(PutUrl) > 0 andalso byte_size(GetUrl) > 0; 425: has_put_and_get_fields(_Elem) -> 426: false. 427: 428: path_ends_with(UrlType, Filename, Result) -> 429: Url = extract_url(Result, UrlType), 430: #{ path := Path } = uri_string:parse(Url), 431: FilenameSize = byte_size(Filename), 432: ReverseFilename = reverse(Filename), 433: case reverse(Path) of 434: <<ReverseFilename:FilenameSize/binary, _/binary>> -> true; 435: _ -> false 436: end. 437: 438: url_contains(UrlType, Filename, Result) -> 439: Url = extract_url(Result, UrlType), 440: binary:match(Url, Filename) =/= nomatch. 441: 442: urls_not_equal(Result) -> 443: Get = extract_url(Result, <<"get">>), 444: Put = extract_url(Result, <<"put">>), 445: Get =/= Put. 446: 447: reverse(List) when is_list(List) -> 448: list_to_binary(lists:reverse(List)); 449: reverse(Binary) -> 450: reverse(binary_to_list(Binary)). 451: 452: upload_service(Client) -> 453: <<"upload.", (escalus_client:server(Client))/binary>>. 454: 455: has_field(Var, Type, Value, Form) -> 456: Fields = Form#xmlel.children, 457: VarFits = fun(I) -> Var =:= undefined orelse exml_query:attr(I, <<"var">>) =:= Var end, 458: TypeFits = fun(I) -> Type =:= undefined orelse exml_query:attr(I, <<"type">>) =:= Type end, 459: ValueFits = 460: fun(I) -> 461: Value =:= undefined orelse 462: Value =:= exml_query:path(I, [{element, <<"value">>}, cdata]) 463: end, 464: lists:any(fun(Item) -> VarFits(Item) andalso TypeFits(Item) andalso ValueFits(Item) end, 465: Fields). 466: 467: extract_url(Result, UrlType) -> 468: exml_query:path(Result, [{element, <<"slot">>}, {element, UrlType}, {attr, <<"url">>}]). 469: 470: ns() -> ?NS_HTTP_UPLOAD_030.