From 328c97b597639322cbb79707699df7751a4cc135 Mon Sep 17 00:00:00 2001 From: Yoshihiro Tanaka Date: Sat, 7 Jan 2017 00:36:02 +0000 Subject: [PATCH 1/5] Special character --- src/lhttpc.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lhttpc.erl b/src/lhttpc.erl index e007f3ea..551e078a 100644 --- a/src/lhttpc.erl +++ b/src/lhttpc.erl @@ -24,7 +24,7 @@ %%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %%% ---------------------------------------------------------------------------- -%%% @author Oscar Hellström +%%% @author Oscar Hellström %%% @doc Main interface to the lightweight http client. %%% See {@link request/4}, {@link request/5} and {@link request/6} functions. %%% @end From de7149e3ac233420e6e61000485ebcd1bb7b42c3 Mon Sep 17 00:00:00 2001 From: Yoshihiro Tanaka Date: Mon, 9 Jan 2017 21:17:15 +0000 Subject: [PATCH 2/5] Stop using deprecated 'now/0' function. --- src/lhttpc.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lhttpc.erl b/src/lhttpc.erl index 551e078a..ef753530 100644 --- a/src/lhttpc.erl +++ b/src/lhttpc.erl @@ -339,7 +339,7 @@ request(URL, Method, Hdrs, Body, Timeout, Options) -> headers(), iolist(), pos_integer(), [option()]) -> result(). request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) -> verify_options(Options, []), - ReqId = now(), + ReqId = erlang:unique_integer(), case proplists:is_defined(stream_to, Options) of true -> StreamTo = proplists:get_value(stream_to, Options), From bc1cbf66bccdfa9ee289527b0798435d55fc150f Mon Sep 17 00:00:00 2001 From: Yoshihiro Tanaka Date: Sat, 7 Jan 2017 00:37:58 +0000 Subject: [PATCH 3/5] Add simple_request/8 API. Add this API for clients that do not require asyncronous mode, partial_upload, or timeout. Using this API should save the cost of temprary process creation, one message passing, and one timer. --- .rebar/erlcinfo | Bin 0 -> 456 bytes src/lhttpc.erl | 40 ++- src/lhttpc_client.erl | 27 +- test/lhttpc_dns_tests.erl | 4 +- test/lhttpc_simple_request_tests.erl | 453 +++++++++++++++++++++++++++ test/lhttpc_tests.erl | 39 ++- 6 files changed, 544 insertions(+), 19 deletions(-) create mode 100644 .rebar/erlcinfo create mode 100644 test/lhttpc_simple_request_tests.erl diff --git a/.rebar/erlcinfo b/.rebar/erlcinfo new file mode 100644 index 0000000000000000000000000000000000000000..ae7f65c882b7d5259abbb5b2348362273faa542e GIT binary patch literal 456 zcmV;(0XP1GPyhf2$ar40l-o{&KoEwvrIu=HjXgy#)M$EV3bAb(--ZmZ%3-C*E*jrf z-^ATg=~5RYO}!$(%>4h%ezTtDq7F`o<3~>`XnGn!2o1buh$akEW)hFF*s-Yp5)*7C z_%-EUE%A#QJk8@7HbVLrG(h(>KyU|Bk=(X)6laT@t-Ix}X*!`$>EhommfJ{sOu`4d{u<<zrZ4yDS2x<*#;R(L`C042zMp$Z}gHb75ponW#QM936F8>!7L;2Vk{NBBsi9qac*K3JK z!$r*rhNH5b2vI3-L7~D))tc @@ -355,6 +355,8 @@ request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) -> Pid = spawn_link(lhttpc_client, request, Args), receive {response, ReqId, Pid, R} -> + %% all throw/1 and 'connection_closed', in addition to normal + %% case is mapped here R; {exit, ReqId, Pid, Reason} -> % We would rather want to exit here, instead of letting the @@ -371,6 +373,40 @@ request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) -> end end. +%% @spec (Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> Result +%% Host = string() +%% Port = 1..65535 +%% Ssl = boolean() +%% Path = string() +%% Method = atom() | string() +%% Hdrs = headers() +%% Body = iolist() +%% Options = [option()] +%% Result = result() +%% @doc This interface does not support timeout, partial upload, and +%% stream_to options. The check for `partial_upload' or `stream_to' +%% is not performed, so users have to be aware of it. +%% Host, Port, Ssl, and Path can be obtained using `lhttpc_lib:parse_url/1'. +%% @end +-spec simple_request(string(), 1..65535, true | false, string(), atom() | string(), + headers(), iolist(), [option()]) -> result(). +simple_request(Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> + verify_options(Options, []), + ReqId = undefined, % ReqId is unnecessary here. + Self = undefined, % self() is unnecessary here. + RetVal = + lhttpc_client:get_request_result( + ReqId, Self, Host, Port, Ssl, Path, Method, Hdrs, Body, Options + ), + case RetVal of + {response, _ReqId, _Self, Response} -> + %% success + throw/1 + 'connection_closed' + Response; + {exit, _ReqId, _Self, Reason} -> + %% error/1 runtime cases + exit(Reason) + end. + %% @spec (UploadState :: UploadState, BodyPart :: BodyPart) -> Result %% BodyPart = iolist() | binary() %% Timeout = integer() | infinity diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index 1cf47f00..3a51dcf0 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -32,6 +32,7 @@ -module(lhttpc_client). -export([request/10]). +-export([get_request_result/10]). -include("lhttpc_types.hrl"). @@ -77,16 +78,7 @@ %% Option = {connect_timeout, Milliseconds} %% @end request(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> - Result = try - execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) - catch - Reason -> - {response, ReqId, self(), {error, Reason}}; - error:closed -> - {response, ReqId, self(), {error, connection_closed}}; - error:Error -> - {exit, ReqId, self(), {Error, erlang:get_stacktrace()}} - end, + Result = get_request_result(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options), case Result of {response, _, _, {ok, {no_return, _}}} -> ok; _Else -> From ! Result @@ -96,6 +88,21 @@ request(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> unlink(From), ok. +get_request_result(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> + try + execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) + catch + %% all throw/1 + Reason -> + {response, ReqId, self(), {error, Reason}}; + %% 'error:closed' only + error:closed -> + {response, ReqId, self(), {error, connection_closed}}; + %% all other error/1 runtime errors + error:Error -> + {exit, ReqId, self(), {Error, erlang:get_stacktrace()}} + end. + execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> UploadWindowSize = proplists:get_value(partial_upload, Options), PartialUpload = proplists:is_defined(partial_upload, Options), diff --git a/test/lhttpc_dns_tests.erl b/test/lhttpc_dns_tests.erl index 08adf3c1..5482ca0c 100644 --- a/test/lhttpc_dns_tests.erl +++ b/test/lhttpc_dns_tests.erl @@ -38,7 +38,7 @@ test_lookup_uncached () -> -define(SINGLE, {4, 0, 0, 0}). test_lookup () -> - meck:new(lhttpc_dns, [ passthrough ]), + meck:new(lhttpc_dns, [ passthrough, no_passthrough_cover ]), meck:expect(lhttpc_dns, lookup_uncached, fun(Host) -> %% io:format(standard_error, "host: ~p~n", [ Host ]), @@ -58,7 +58,7 @@ test_lookup () -> meck:expect(lhttpc_dns, os_timestamp, fun () -> { 0, erlang:get(dns_ts), 0 } end), - meck:new(random, [ passthrough, unstick ]), + meck:new(random, [ passthrough, unstick, no_passthrough_cover ]), meck:expect(random, uniform, fun (N) -> erlang:get(dns_random) rem N + 1 end), lhttpc_dns:reset_table(), diff --git a/test/lhttpc_simple_request_tests.erl b/test/lhttpc_simple_request_tests.erl new file mode 100644 index 00000000..2d97f72b --- /dev/null +++ b/test/lhttpc_simple_request_tests.erl @@ -0,0 +1,453 @@ +-module(lhttpc_simple_request_tests). + +-export([test_no/2]). +-export([request/3]). +-export([request/5]). +-import(webserver, [start/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(DEFAULT_STRING, "Great success!"). +test_no(N, Tests) -> + setelement(2, Tests, + setelement(4, element(2, Tests), + lists:nth(N, element(4, element(2, Tests))))). + +%%% Eunit setup stuff + +start_app() -> + lhttpc_tests:start_app(). + +stop_app(_Any) -> + lhttpc_tests:stop_app(_Any). + +tcp_test_() -> + {inorder, + {setup, fun start_app/0, fun stop_app/1, [ + ?_test(simple_get()), + ?_test(empty_get()), + ?_test(get_no_content()), + ?_test(post_no_content()), + ?_test(get_with_mandatory_hdrs()), + ?_test(get_with_connect_options()), + ?_test(no_content_length()), + ?_test(no_content_length_1_0()), + ?_test(get_not_modified()), + ?_test(simple_head()), + ?_test(simple_head_atom()), + ?_test(delete_no_content()), + ?_test(delete_content()), + ?_test(options_content()), + ?_test(options_no_content()), + ?_test(server_connection_close()), + ?_test(client_connection_close()), + ?_test(pre_1_1_server_connection()), + ?_test(pre_1_1_server_keep_alive()), + ?_test(simple_put()), + ?_test(post()), + ?_test(post_100_continue()), + ?_test(bad_url()), + ?_test(persistent_connection()), + ?_test(connection_timeout()), + ?_test(chunked_encoding()), + ?_test(close_connection()), + ?_test(message_queue()) + ]} + }. + +ssl_test_() -> + {inorder, + {setup, fun start_app/0, fun stop_app/1, [ + ?_test(ssl_get()), + ?_test(ssl_post()), + ?_test(ssl_chunked()) + ]} + }. + +other_test_() -> + [ + ?_test(invalid_options()) + ]. + +request(URL, Method, Hdrs) -> + {Host, Port, Path, Ssl} = lhttpc_lib:parse_url(URL), + lhttpc:simple_request(Host, Port, Ssl, Path, Method, Hdrs, [], []). + +request(URL, Method, Hdrs, Body, Options) -> + {Host, Port, Path, Ssl} = lhttpc_lib:parse_url(URL), + lhttpc:simple_request(Host, Port, Ssl, Path, Method, Hdrs, Body, Options). + +%%% Tests + +message_queue() -> + receive X -> erlang:error({unexpected_message, X}) after 0 -> ok end. + +simple_get() -> + simple(get), + simple("GET"). + +empty_get() -> + Port = start(gen_tcp, [fun empty_body/5]), + URL = url(Port, "/empty_get"), + {ok, Response} = ?MODULE:request(URL, "GET", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +get_no_content() -> + no_content(get, 2). + +post_no_content() -> + no_content("POST", 3). + +get_with_mandatory_hdrs() -> + Port = start(gen_tcp, [fun simple_response/5]), + URL = url(Port, "/get_with_mandatory_hdrs"), + Body = <>, + Hdrs = [ + {"content-length", integer_to_list(size(Body))}, + {"host", "localhost"} + ], + {ok, Response} = ?MODULE:request(URL, "POST", Hdrs, Body, []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)). + +get_with_connect_options() -> + Port = start(gen_tcp, [fun empty_body/5]), + URL = url(Port, "/get_with_connect_options"), + Options = [{connect_options, [{ip, {127, 0, 0, 1}}, {port, 0}]}], + {ok, Response} = ?MODULE:request(URL, "GET", [], [], Options), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +no_content_length() -> + Port = start(gen_tcp, [fun no_content_length/5]), + URL = url(Port, "/no_content_length"), + {ok, Response} = ?MODULE:request(URL, "GET", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)). + +no_content_length_1_0() -> + Port = start(gen_tcp, [fun no_content_length_1_0/5]), + URL = url(Port, "/no_content_length_1_0"), + {ok, Response} = ?MODULE:request(URL, "GET", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)). + +get_not_modified() -> + Port = start(gen_tcp, [fun not_modified_response/5]), + URL = url(Port, "/get_not_modified"), + {ok, Response} = ?MODULE:request(URL, "GET", [], [], []), + ?assertEqual({304, "Not Modified"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +simple_head() -> + Port = start(gen_tcp, [fun head_response/5]), + URL = url(Port, "/simple_head"), + {ok, Response} = ?MODULE:request(URL, "HEAD", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +simple_head_atom() -> + Port = start(gen_tcp, [fun head_response/5]), + URL = url(Port, "/simple_head_atom"), + {ok, Response} = ?MODULE:request(URL, head, []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +delete_no_content() -> + Port = start(gen_tcp, [fun no_content_response/5]), + URL = url(Port, "/delete_no_content"), + {ok, Response} = ?MODULE:request(URL, delete, []), + ?assertEqual({204, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +delete_content() -> + Port = start(gen_tcp, [fun simple_response/5]), + URL = url(Port, "/delete_content"), + {ok, Response} = ?MODULE:request(URL, "DELETE", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)). + +options_no_content() -> + Port = start(gen_tcp, [fun head_response/5]), + URL = url(Port, "/options_no_content"), + {ok, Response} = ?MODULE:request(URL, "OPTIONS", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)). + +options_content() -> + Port = start(gen_tcp, [fun simple_response/5]), + URL = url(Port, "/options_content"), + {ok, Response} = ?MODULE:request(URL, "OPTIONS", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)). + +server_connection_close() -> + Port = start(gen_tcp, [fun respond_and_close/5]), + URL = url(Port, "/server_connection_close"), + Body = pid_to_list(self()), + {ok, Response} = ?MODULE:request(URL, "PUT", [], Body, []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)), + receive closed -> ok end. + +client_connection_close() -> + Port = start(gen_tcp, [fun respond_and_wait/5]), + URL = url(Port, "/client_connection_close"), + Body = pid_to_list(self()), + Hdrs = [{"Connection", "close"}], + {ok, _} = ?MODULE:request(URL, put, Hdrs, Body, []), + % Wait for the server to see that socket has been closed + receive closed -> ok end. + +pre_1_1_server_connection() -> + Port = start(gen_tcp, [fun pre_1_1_server/5]), + URL = url(Port, "/pre_1_1_server_connection"), + Body = pid_to_list(self()), + {ok, _} = ?MODULE:request(URL, put, [], Body, []), + % Wait for the server to see that socket has been closed. + % The socket should be closed by us since the server responded with a + % 1.0 version, and not the Connection: keep-alive header. + receive closed -> ok end. + +pre_1_1_server_keep_alive() -> + Port = start(gen_tcp, [ + fun pre_1_1_server_keep_alive/5, + fun pre_1_1_server/5 + ]), + URL = url(Port, "/pre_1_1_server_keep_alive"), + Body = pid_to_list(self()), + {ok, Response1} = ?MODULE:request(URL, get, [], [], []), + {ok, Response2} = ?MODULE:request(URL, put, [], Body, []), + ?assertEqual({200, "OK"}, status(Response1)), + ?assertEqual({200, "OK"}, status(Response2)), + ?assertEqual(<>, body(Response1)), + ?assertEqual(<>, body(Response2)), + % Wait for the server to see that socket has been closed. + % The socket should be closed by us since the server responded with a + % 1.0 version, and not the Connection: keep-alive header. + receive closed -> ok end. + +simple_put() -> + simple(put), + simple("PUT"). + +post() -> + Port = start(gen_tcp, [fun copy_body/5]), + URL = url(Port, "/post"), + {X, Y, Z} = os:timestamp(), + Body = [ + "This is a rather simple post :)", + integer_to_list(X), + integer_to_list(Y), + integer_to_list(Z) + ], + {ok, Response} = ?MODULE:request(URL, "POST", [], Body, []), + {StatusCode, ReasonPhrase} = status(Response), + ?assertEqual(200, StatusCode), + ?assertEqual("OK", ReasonPhrase), + ?assertEqual(iolist_to_binary(Body), body(Response)). + +post_100_continue() -> + Port = start(gen_tcp, [fun copy_body_100_continue/5]), + URL = url(Port, "/post_100_continue"), + {X, Y, Z} = os:timestamp(), + Body = [ + "This is a rather simple post :)", + integer_to_list(X), + integer_to_list(Y), + integer_to_list(Z) + ], + {ok, Response} = ?MODULE:request(URL, "POST", [], Body, []), + {StatusCode, ReasonPhrase} = status(Response), + ?assertEqual(200, StatusCode), + ?assertEqual("OK", ReasonPhrase), + ?assertEqual(iolist_to_binary(Body), body(Response)). + +bad_url() -> + ?assertError(_, ?MODULE:request(ost, "GET", [])). + +persistent_connection() -> + Port = start(gen_tcp, [ + fun simple_response/5, + fun simple_response/5, + fun copy_body/5 + ]), + URL = url(Port, "/persistent_connection"), + {ok, FirstResponse} = ?MODULE:request(URL, "GET", []), + Headers = [{"KeepAlive", "300"}], % shouldn't be needed + {ok, SecondResponse} = ?MODULE:request(URL, "GET", Headers), + {ok, ThirdResponse} = ?MODULE:request(URL, "POST", []), + ?assertEqual({200, "OK"}, status(FirstResponse)), + ?assertEqual(<>, body(FirstResponse)), + ?assertEqual({200, "OK"}, status(SecondResponse)), + ?assertEqual(<>, body(SecondResponse)), + ?assertEqual({200, "OK"}, status(ThirdResponse)), + ?assertEqual(<<>>, body(ThirdResponse)). + +connection_timeout() -> + Port = start(gen_tcp, [fun simple_response/5, fun simple_response/5]), + URL = url(Port, "/connection_timeout"), + {ok, Response} = ?MODULE:request(URL, get, [], [], [ {connection_timeout, 50} ]), + ?assertEqual({0,1}, lhttpc_lb:connection_count("localhost", Port, false)), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)), + timer:sleep(100), + ?assertEqual({0,0}, lhttpc_lb:connection_count("localhost", Port, false)). + +chunked_encoding() -> + Port = start(gen_tcp, [fun chunked_response/5, fun chunked_response_t/5]), + URL = url(Port, "/chunked_encoding"), + {ok, FirstResponse} = ?MODULE:request(URL, get, []), + ?assertEqual({200, "OK"}, status(FirstResponse)), + ?assertEqual(<>, body(FirstResponse)), + ?assertEqual("chunked", lhttpc_lib:header_value("transfer-encoding", + headers(FirstResponse))), + {ok, SecondResponse} = ?MODULE:request(URL, get, []), + ?assertEqual({200, "OK"}, status(SecondResponse)), + ?assertEqual(<<"Again, great success!">>, body(SecondResponse)), + ?assertEqual("ChUnKeD", lhttpc_lib:header_value("transfer-encoding", + headers(SecondResponse))), + ?assertEqual("1", lhttpc_lib:header_value("trailer-1", + headers(SecondResponse))), + ?assertEqual("2", lhttpc_lib:header_value("trailer-2", + headers(SecondResponse))). + +close_connection() -> + %receive _ -> ok after 0 -> ok end, + Port = start(gen_tcp, [fun close_connection/5]), + URL = url(Port, "/close"), + ?assertEqual({error, connection_closed}, ?MODULE:request(URL, "GET", [])). + +ssl_get() -> + Port = start(ssl, [fun simple_response/5]), + URL = ssl_url(Port, "/ssl_get"), + {ok, Response} = ?MODULE:request(URL, "GET", []), + ?assertEqual({200, "OK"}, status(Response)), + ?assertEqual(<>, body(Response)). + +ssl_post() -> + Port = start(ssl, [fun copy_body/5]), + URL = ssl_url(Port, "/ssl_post"), + Body = "SSL Test + Port = start(ssl, [fun chunked_response/5, fun chunked_response_t/5]), + URL = ssl_url(Port, "/ssl_chunked"), + FirstResult = ?MODULE:request(URL, get, []), + ?assertMatch({ok, _}, FirstResult), + {ok, FirstResponse} = FirstResult, + ?assertEqual({200, "OK"}, status(FirstResponse)), + ?assertEqual(<>, body(FirstResponse)), + ?assertEqual("chunked", lhttpc_lib:header_value("transfer-encoding", + headers(FirstResponse))), + SecondResult = ?MODULE:request(URL, get, []), + {ok, SecondResponse} = SecondResult, + ?assertEqual({200, "OK"}, status(SecondResponse)), + ?assertEqual(<<"Again, great success!">>, body(SecondResponse)), + ?assertEqual("ChUnKeD", lhttpc_lib:header_value("transfer-encoding", + headers(SecondResponse))), + ?assertEqual("1", lhttpc_lib:header_value("Trailer-1", + headers(SecondResponse))), + ?assertEqual("2", lhttpc_lib:header_value("Trailer-2", + headers(SecondResponse))). + +invalid_options() -> + ?assertError({bad_options, [{foo, bar}, bad_option]}, + ?MODULE:request("http://localhost/", get, [], <<>>, + [bad_option, {foo, bar}])). + +%%% Helpers functions + +simple(Method) -> + Port = start(gen_tcp, [fun simple_response/5]), + URL = url(Port, "/simple"), + {ok, Response} = ?MODULE:request(URL, Method, []), + {StatusCode, ReasonPhrase} = status(Response), + ?assertEqual(200, StatusCode), + ?assertEqual("OK", ReasonPhrase), + ?assertEqual(<>, body(Response)). + +no_content(Method, Count) -> + Responses = lists:duplicate(Count, fun no_content_response/5), + Port = start(gen_tcp, Responses), + URL = url(Port, "/" ++ lhttpc_lib:maybe_atom_to_list(Method) ++ "_no_content"), + lists:foreach( + fun (_) -> + {ok, Response} = ?MODULE:request(URL, Method, [], <<>>, + [ {connect_timeout, 100}, + {connection_timeout, 5000}, + {max_connections, 3} ]), + ?assertEqual({204, "OK"}, status(Response)), + ?assertEqual(<<>>, body(Response)), + timer:sleep(100) + end, Responses). + +url(Port, Path) -> + lhttpc_tests:url(Port, Path). + +ssl_url(Port, Path) -> + lhttpc_tests:ssl_url(Port, Path). + +status({Status, _, _}) -> + Status. + +body({_, _, Body}) -> + Body. + +headers({_, Headers, _}) -> + Headers. + +%%% Responders +simple_response(Module, Socket, _Request, _Headers, Body) -> + lhttpc_tests:simple_response(Module, Socket, _Request, _Headers, Body). + +head_response(Module, Socket, _Request, _Headers, _Body) -> + lhttpc_tests:head_response(Module, Socket, _Request, _Headers, _Body). + +no_content_response(Module, Socket, _Request, _Headers, _Body) -> + lhttpc_tests:no_content_response(Module, Socket, _Request, _Headers, _Body). + +empty_body(Module, Socket, _A, _B, _C) -> + lhttpc_tests:empty_body(Module, Socket, _A, _B, _C). + +copy_body(Module, Socket, _A, _B, Body) -> + lhttpc_tests:copy_body(Module, Socket, _A, _B, Body). + +copy_body_100_continue(Module, Socket, _A, _B, Body) -> + lhttpc_tests:copy_body_100_continue(Module, Socket, _A, _B, Body). + +respond_and_close(Module, Socket, _A, _B, Body) -> + lhttpc_tests:respond_and_close(Module, Socket, _A, _B, Body). + +respond_and_wait(Module, Socket, _A, _B, Body) -> + lhttpc_tests:respond_and_wait(Module, Socket, _A, _B, Body). + +pre_1_1_server(Module, Socket, _A, _B, Body) -> + lhttpc_tests:pre_1_1_server(Module, Socket, _A, _B, Body). + +pre_1_1_server_keep_alive(Module, Socket, _A, _B, _C) -> + lhttpc_tests:pre_1_1_server_keep_alive(Module, Socket, _A, _B, _C). + +no_content_length(Module, Socket, _A, _B, _C) -> + lhttpc_tests:no_content_length(Module, Socket, _A, _B, _C). + +no_content_length_1_0(Module, Socket, _A, _B, _C) -> + lhttpc_tests:no_content_length_1_0(Module, Socket, _A, _B, _C). + +chunked_response(Module, Socket, _A, _B, _C) -> + lhttpc_tests:chunked_response(Module, Socket, _A, _B, _C). + +chunked_response_t(Module, Socket, _A, _B, _C) -> + lhttpc_tests:chunked_response_t(Module, Socket, _A, _B, _C). + +close_connection(Module, Socket, _A, _B, _C) -> + lhttpc_tests:close_connection(Module, Socket, _A, _B, _C). + +not_modified_response(Module, Socket, _Request, _Headers, _Body) -> + lhttpc_tests:not_modified_response(Module, Socket, _Request, _Headers, _Body). + diff --git a/test/lhttpc_tests.erl b/test/lhttpc_tests.erl index 1d1f1e0c..b3206e60 100644 --- a/test/lhttpc_tests.erl +++ b/test/lhttpc_tests.erl @@ -28,6 +28,31 @@ -module(lhttpc_tests). -export([test_no/2]). + +-export( % Shared with `lhttpc_simple_request_tests'. + [ + chunked_response/5 + , chunked_response_t/5 + , close_connection/5 + , copy_body/5 + , copy_body_100_continue/5 + , empty_body/5 + , head_response/5 + , no_content_length/5 + , no_content_length_1_0/5 + , no_content_response/5 + , not_modified_response/5 + , pre_1_1_server/5 + , pre_1_1_server_keep_alive/5 + , respond_and_close/5 + , respond_and_wait/5 + , simple_response/5 + , ssl_url/2 + , start_app/0 + , stop_app/1 + , url/2 + ]). + -import(webserver, [start/2]). -include_lib("eunit/include/eunit.hrl"). @@ -100,9 +125,12 @@ test_no(N, Tests) -> start_app() -> application:start(crypto), + application:start(asn1), application:start(public_key), ok = application:start(ssl), - ok = lhttpc:start(). + ok = lhttpc:start(), + timer:sleep(1000), + ok. stop_app(_) -> timer:sleep(1000), @@ -334,7 +362,7 @@ simple_put() -> post() -> Port = start(gen_tcp, [fun copy_body/5]), URL = url(Port, "/post"), - {X, Y, Z} = now(), + {X, Y, Z} = os:timestamp(), Body = [ "This is a rather simple post :)", integer_to_list(X), @@ -350,7 +378,7 @@ post() -> post_100_continue() -> Port = start(gen_tcp, [fun copy_body_100_continue/5]), URL = url(Port, "/post_100_continue"), - {X, Y, Z} = now(), + {X, Y, Z} = os:timestamp(), Body = [ "This is a rather simple post :)", integer_to_list(X), @@ -647,6 +675,7 @@ partial_download_slow_chunks() -> ?assertEqual(<>, Body). close_connection() -> + %receive _ -> ok after 0 -> ok end, Port = start(gen_tcp, [fun close_connection/5]), URL = url(Port, "/close"), ?assertEqual({error, connection_closed}, lhttpc:request(URL, "GET", [], @@ -671,14 +700,14 @@ ssl_post() -> ssl_chunked() -> Port = start(ssl, [fun chunked_response/5, fun chunked_response_t/5]), URL = ssl_url(Port, "/ssl_chunked"), - FirstResult = lhttpc:request(URL, get, [], 100), + FirstResult = lhttpc:request(URL, get, [], 1000), ?assertMatch({ok, _}, FirstResult), {ok, FirstResponse} = FirstResult, ?assertEqual({200, "OK"}, status(FirstResponse)), ?assertEqual(<>, body(FirstResponse)), ?assertEqual("chunked", lhttpc_lib:header_value("transfer-encoding", headers(FirstResponse))), - SecondResult = lhttpc:request(URL, get, [], 100), + SecondResult = lhttpc:request(URL, get, [], 1000), {ok, SecondResponse} = SecondResult, ?assertEqual({200, "OK"}, status(SecondResponse)), ?assertEqual(<<"Again, great success!">>, body(SecondResponse)), From 4dbee9b348b5b94401c2c01650321de3c1cde32e Mon Sep 17 00:00:00 2001 From: Yoshihiro Tanaka Date: Sat, 14 Jan 2017 01:02:45 +0000 Subject: [PATCH 4/5] Remove .rebar/erlcinfo --- .rebar/erlcinfo | Bin 456 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .rebar/erlcinfo diff --git a/.rebar/erlcinfo b/.rebar/erlcinfo deleted file mode 100644 index ae7f65c882b7d5259abbb5b2348362273faa542e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 456 zcmV;(0XP1GPyhf2$ar40l-o{&KoEwvrIu=HjXgy#)M$EV3bAb(--ZmZ%3-C*E*jrf z-^ATg=~5RYO}!$(%>4h%ezTtDq7F`o<3~>`XnGn!2o1buh$akEW)hFF*s-Yp5)*7C z_%-EUE%A#QJk8@7HbVLrG(h(>KyU|Bk=(X)6laT@t-Ix}X*!`$>EhommfJ{sOu`4d{u<<zrZ4yDS2x<*#;R(L`C042zMp$Z}gHb75ponW#QM936F8>!7L;2Vk{NBBsi9qac*K3JK z!$r*rhNH5b2vI3-L7~D))tc Date: Sat, 14 Jan 2017 01:11:14 +0000 Subject: [PATCH 5/5] Improve performance of header processing According to the result of profiling, 'lhttpc_lib:header_value/2,3' seems to be one of the hot spot. This commit tries to reduce the overhead of calling this function. To send a request, it can be skipped completely if a user passes new flags 'is_host_defined' and 'is_content_length_defined'. To receive a request, it's difficult to skip the call completely but the performance of this function should be better by ~3 times. --- include/lhttpc_types.hrl | 4 +- src/lhttpc.erl | 9 +++ src/lhttpc_client.erl | 6 +- src/lhttpc_lib.erl | 148 +++++++++++++++++++++++++------------- test/lhttpc_lib_tests.erl | 31 ++++++++ test/lhttpc_tests.erl | 31 ++++++-- 6 files changed, 173 insertions(+), 56 deletions(-) diff --git a/include/lhttpc_types.hrl b/include/lhttpc_types.hrl index 3455d9cb..e14ab2ec 100644 --- a/include/lhttpc_types.hrl +++ b/include/lhttpc_types.hrl @@ -37,7 +37,9 @@ {send_retry, non_neg_integer()} | {stream_to, pid()} | {partial_upload, non_neg_integer() | infinity} | - {partial_download, pid(), non_neg_integer() | infinity}. + {partial_download, pid(), non_neg_integer() | infinity} | + {is_host_defined, boolean()} | + {is_content_length_defined, boolean()}. -type options() :: [option()]. diff --git a/src/lhttpc.erl b/src/lhttpc.erl index 6ef0e890..e7778ff2 100644 --- a/src/lhttpc.erl +++ b/src/lhttpc.erl @@ -334,6 +334,11 @@ request(URL, Method, Hdrs, Body, Timeout, Options) -> %% lhttpc will never close the connection itself; it will only be %% closed by the other end or the TCP keepalive mechanism. %% +%% `{is_content_length_defined, Bool}' and `{is_host_defined, Bool}' +%% is used to skip lhttpc_lib:header_value/2,3. +%% When boolean value is specified, there is no need to check if the +%% value exists. +%% %% @end -spec request(string(), 1..65535, true | false, string(), atom() | string(), headers(), iolist(), pos_integer(), [option()]) -> result(). @@ -633,6 +638,10 @@ verify_options([{connect_options, List} | Options], Errors) verify_options(Options, Errors); verify_options([{stream_to, Pid} | Options], Errors) when is_pid(Pid) -> verify_options(Options, Errors); +verify_options([{is_content_length_defined, Bool} | Options], Errors) when is_boolean(Bool) -> + verify_options(Options, Errors); +verify_options([{is_host_defined, Bool} | Options], Errors) when is_boolean(Bool) -> + verify_options(Options, Errors); verify_options([Option | Options], Errors) -> verify_options(Options, [Option | Errors]); verify_options([], Errors) -> diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index 3a51dcf0..17cca378 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -111,8 +111,11 @@ execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> NormalizedMethod = lhttpc_lib:normalize_method(Method), MaxConnections = proplists:get_value(max_connections, Options, 10), ConnectionTimeout = proplists:get_value(connection_timeout, Options, infinity), + IsContentLengthDefined = proplists:get_value(is_content_length_defined, Options, undefined), + IsHostDefined = proplists:get_value(is_host_defined, Options, undefined), + {ChunkedUpload, Request} = lhttpc_lib:format_request(Path, NormalizedMethod, - Hdrs, Host, Port, Body, PartialUpload), + Hdrs, Host, Port, Body, PartialUpload, IsContentLengthDefined, IsHostDefined), Socket = case lhttpc_lb:checkout(Host, Port, Ssl, MaxConnections, ConnectionTimeout) of {ok, S} -> S; % Re-using HTTP/1.1 connections retry_later -> throw(retry_later); @@ -343,6 +346,7 @@ has_body(_, 304, _) -> has_body(_, _, _) -> true. % All other responses are assumed to have a body +-spec body_type(Hdrs::list({string(), term()})) -> chunked | infinite | {fixed_length, integer()}. body_type(Hdrs) -> % Find out how to read the entity body from the request. % * If we have a Content-Length, just use that and read the complete diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index b5687add..e660997d 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -32,7 +32,7 @@ -export([ parse_url/1, - format_request/7, + format_request/9, header_value/2, header_value/3, normalize_method/1 @@ -42,6 +42,36 @@ -export([format_hdrs/1, dec/1]). -include("lhttpc_types.hrl"). +-define(L(C), + case C of + $A -> $a; + $B -> $b; + $C -> $c; + $D -> $d; + $E -> $e; + $F -> $f; + $G -> $g; + $H -> $h; + $I -> $i; + $J -> $j; + $K -> $k; + $L -> $l; + $M -> $m; + $N -> $n; + $O -> $o; + $P -> $p; + $Q -> $q; + $R -> $r; + $S -> $s; + $T -> $t; + $U -> $u; + $V -> $v; + $W -> $w; + $X -> $x; + $Y -> $y; + $Z -> $z; + C -> C + end). %% @spec header_value(Header, Headers) -> undefined | term() %% Header = string() @@ -68,14 +98,23 @@ header_value(Hdr, Hdrs) -> %% @end -spec header_value(string(), [{string(), Value}], Default) -> Default | Value. -header_value(Hdr, [{Hdr, Value} | _], _) -> +header_value(Hdr, Hdrs, Default) -> + header_value_search(Hdr, Hdr, Hdrs, Default). + +header_value_search(Hdr, Hdr, [{Hdr, Value} | _], _) -> + Value; +header_value_search([], _Hdr, [{[], Value}| _Hdrs], _Default) -> Value; -header_value(Hdr, [{ThisHdr, Value}| Hdrs], Default) -> - case string:equal(string:to_lower(ThisHdr), Hdr) of - true -> Value; - false -> header_value(Hdr, Hdrs, Default) +header_value_search(_, Hdr, [{[], _}| Hdrs], Default) -> + header_value_search(Hdr, Hdr, Hdrs, Default); +header_value_search([], Hdr, [_| Hdrs], Default) -> + header_value_search(Hdr, Hdr, Hdrs, Default); +header_value_search([H0|Hdr0], Hdr, [{[H1|Hdr1], Value}| Hdrs], Default) -> + case H0 == ?L(H1) of + true -> header_value_search(Hdr0, Hdr, [{Hdr1, Value}| Hdrs], Default); + false -> header_value_search(Hdr, Hdr, Hdrs, Default) end; -header_value(_, [], Default) -> +header_value_search(_, _Hdr, [], Default) -> Default. %% @spec (Item) -> OtherItem @@ -151,11 +190,20 @@ split_port(Scheme, [P | T], Port) -> %% Port = integer() %% Body = iolist() %% PartialUpload = true | false +%% IsContentLengthUnDefined = true | false -spec format_request(iolist(), atom() | string(), headers(), string(), - integer(), iolist(), true | false ) -> {true | false, iolist()}. -format_request(Path, Method, Hdrs, Host, Port, Body, PartialUpload) -> - AllHdrs = add_mandatory_hdrs(Method, Hdrs, Host, Port, Body, PartialUpload), - IsChunked = is_chunked(AllHdrs), + integer(), iolist(), true | false, boolean() | 'undefined', boolean() | 'undefined') -> {true | false, iolist()}. +format_request(Path, Method, Hdrs, Host, Port, Body, PartialUpload, IsContentLengthDefined, IsHostDefined) -> + {IsChunked, ContentHdrs} = + if + Method =/= "POST" andalso Method =/= "PUT" -> {false, Hdrs}; + PartialUpload == false -> + {false, add_content_length(Hdrs, Body, IsContentLengthDefined)}; + PartialUpload == true -> + add_transfer_encoding(Hdrs) + end, + + AllHdrs = add_host(ContentHdrs, Host, Port, IsHostDefined), { IsChunked, [ @@ -203,42 +251,48 @@ format_body(Body, true) -> ] end. -add_mandatory_hdrs(Method, Hdrs, Host, Port, Body, PartialUpload) -> - ContentHdrs = add_content_headers(Method, Hdrs, Body, PartialUpload), - add_host(ContentHdrs, Host, Port). +-spec add_content_length(Hdrs::headers(), Body::iolist(), IsContentLengthDefined::boolean()) -> headers(). +add_content_length(Hdrs, _Body, true) -> Hdrs; +add_content_length(Hdrs, Body, false) -> add_content_length(Hdrs, Body); +add_content_length(Hdrs, Body, undefined) -> + case header_value("content-length", Hdrs) of + undefined -> add_content_length(Hdrs, Body); + _ -> Hdrs % We have a content length + end. -add_content_headers("POST", Hdrs, Body, PartialUpload) -> - add_content_headers(Hdrs, Body, PartialUpload); -add_content_headers("PUT", Hdrs, Body, PartialUpload) -> - add_content_headers(Hdrs, Body, PartialUpload); -add_content_headers(_, Hdrs, _, _PartialUpload) -> - Hdrs. +add_content_length(Hdrs, Body) -> + ContentLength = integer_to_list(iolist_size(Body)), + [{"Content-Length", ContentLength} | Hdrs]. -add_content_headers(Hdrs, Body, false) -> - case header_value("content-length", Hdrs) of - undefined -> - ContentLength = integer_to_list(iolist_size(Body)), - [{"Content-Length", ContentLength} | Hdrs]; - _ -> % We have a content length - Hdrs - end; -add_content_headers(Hdrs, _Body, true) -> - case {header_value("content-length", Hdrs), - header_value("transfer-encoding", Hdrs)} of - {undefined, undefined} -> - [{"Transfer-Encoding", "chunked"} | Hdrs]; - {undefined, TransferEncoding} -> - case string:to_lower(TransferEncoding) of - "chunked" -> Hdrs; - _ -> erlang:error({error, unsupported_transfer_encoding}) - end; - {_Length, undefined} -> - Hdrs; - {_Length, _TransferEncoding} -> %% have both cont.length and chunked + +-define(CHUNKED, true). +-define(NOT_CHUNKED, false). +-spec add_transfer_encoding(Hdrs::headers()) -> {IsChunked::boolean(), Hdrs::headers()}. +add_transfer_encoding(Hdrs) -> + case {header_value("content-length", Hdrs), header_value("transfer-encoding", Hdrs)} of + {undefined, undefined} -> {?CHUNKED, [{"Transfer-Encoding", "chunked"} | Hdrs]}; + {undefined, TransferEncoding} -> {?CHUNKED, confirm_chunked(TransferEncoding, Hdrs)}; + {_Length, undefined} -> {?NOT_CHUNKED, Hdrs}; + _Else -> + %% Have both cont.length and chunked. This can happen + %% regardless of `is_content_length_defined' flag erlang:error({error, bad_header}) end. -add_host(Hdrs, Host, Port) -> +-spec confirm_chunked(TransferEncoding::string(), Hdrs::headers()) -> Hdrs::headers(). +confirm_chunked(TransferEncoding, Hdrs) -> + case string:to_lower(TransferEncoding) of + "chunked" -> Hdrs; + _ -> erlang:error({error, unsupported_transfer_encoding}) + end. + + +-spec add_host(Hdrs::headers(), Host::host(), Port::non_neg_integer(), IsHostDefined::boolean()) -> + Hdrs::headers(). +add_host(Hdrs, _Host, _Port, true) -> Hdrs; +add_host(Hdrs, Host, Port, false) -> + [{"Host", host(Host, Port) } | Hdrs]; +add_host(Hdrs, Host, Port, 'undefined') -> case header_value("host", Hdrs) of undefined -> [{"Host", host(Host, Port) } | Hdrs]; @@ -246,17 +300,11 @@ add_host(Hdrs, Host, Port) -> Hdrs end. -is_chunked(Hdrs) -> - TransferEncoding = string:to_lower( - header_value("transfer-encoding", Hdrs, "undefined")), - case TransferEncoding of - "chunked" -> true; - _ -> false - end. - -spec dec(timeout()) -> timeout(). dec(Num) when is_integer(Num) -> Num - 1; dec(Else) -> Else. +-spec host(Host::string(), Port::port()) -> Host::string() | list(). host(Host, 80) -> Host; host(Host, Port) -> [Host, $:, integer_to_list(Port)]. + diff --git a/test/lhttpc_lib_tests.erl b/test/lhttpc_lib_tests.erl index b1dfe7d3..76d4046a 100644 --- a/test/lhttpc_lib_tests.erl +++ b/test/lhttpc_lib_tests.erl @@ -54,3 +54,34 @@ parse_url_test_() -> ?_assertEqual({"host", 180, "?query", false}, lhttpc_lib:parse_url("http://host:180?query")) ]. + + +header_value_test_() -> + Hdrs = [ + {"Accept-Encoding","gzip"} + ,{"Access-Control-Allow-Methods","GET, POST, OPTIONS"} + ,{"Bid-Request-End-Time","1482516525448"} + ,{"Bid-Request-Start-Time","1482516525430"} + ,{"Content-Encoding","gzip"} + ,{"Content-Length","1034"} + ,{"Expires","Tue, 11 Oct 1977 12:34:56 GMT"} + ,{"Host","10.5.75.249"} + ,{"P3p","CP=\"NON DEVa PSAa PSDa OUR NOR NAV\",policyref=\"/w3c/p3p.xml\""} + ,{"X-Frame-Options","SAMEORIGIN"} + ,{"X-OpenX-Id","0050ac11-8170-189d-2915-5e00ce817498"} + ,{"X-OpenX-Rtb","b7f40ed1-c62a-11e6-bfe1-005056a21a26"} + ,{"accept","application/json"} + ,{"content-type","application/json"} + ,{"x-openrtb-version","2.4"} + ,{"ABCDEFGHIJKLMNOPQRSTUVWXYZ-","alpha-minus"} + ], + [ + ?_assertEqual("1034", lhttpc_lib:header_value("content-length", Hdrs, undefined)) + ,?_assertEqual("10.5.75.249", lhttpc_lib:header_value("host", Hdrs, undefined)) + ,?_assertEqual("CP=\"NON DEVa PSAa PSDa OUR NOR NAV\",policyref=\"/w3c/p3p.xml\"", lhttpc_lib:header_value("p3p", Hdrs, undefined)) + ,?_assertEqual(undefined, lhttpc_lib:header_value("content", Hdrs, undefined)) + ,?_assertEqual(undefined, lhttpc_lib:header_value("content-length1", Hdrs, undefined)) + ,?_assertEqual(undefined, lhttpc_lib:header_value("content-length1", [], undefined)) + ,?_assertEqual(undefined, lhttpc_lib:header_value([], [], undefined)) + ,?_assertEqual("alpha-minus", lhttpc_lib:header_value("abcdefghijklmnopqrstuvwxyz-", Hdrs, undefined)) + ]. diff --git a/test/lhttpc_tests.erl b/test/lhttpc_tests.erl index b3206e60..07608871 100644 --- a/test/lhttpc_tests.erl +++ b/test/lhttpc_tests.erl @@ -376,7 +376,9 @@ post() -> ?assertEqual(iolist_to_binary(Body), body(Response)). post_100_continue() -> - Port = start(gen_tcp, [fun copy_body_100_continue/5]), + Port = start(gen_tcp, [fun copy_body_100_continue/5, + fun copy_body_100_continue/5, + fun copy_body_100_continue/5]), URL = url(Port, "/post_100_continue"), {X, Y, Z} = os:timestamp(), Body = [ @@ -387,9 +389,30 @@ post_100_continue() -> ], {ok, Response} = lhttpc:request(URL, "POST", [], Body, 1000), {StatusCode, ReasonPhrase} = status(Response), - ?assertEqual(200, StatusCode), - ?assertEqual("OK", ReasonPhrase), - ?assertEqual(iolist_to_binary(Body), body(Response)). + ?assertEqual({200, "OK"}, {StatusCode, ReasonPhrase}), + ?assertEqual(iolist_to_binary(Body), body(Response)), + + %% With options false + {ok, Response2} = lhttpc:request(URL, "POST", [], Body, 1000, + [{is_content_length_defined, false}, + {is_host_defined, false} + ]), + {StatusCode2, ReasonPhrase2} = status(Response2), + ?assertEqual({200, "OK"}, {StatusCode2, ReasonPhrase2}), + ?assertEqual(iolist_to_binary(Body), body(Response2)), + + + %% With options true + ContentLength = integer_to_list(iolist_size(Body)), + Hdrs = [{"Content-Length", ContentLength}, {"Host", "localhost"}], + {ok, Response3} = lhttpc:request(URL, "POST", Hdrs, Body, 1000, + [{is_content_length_defined, true}, + {is_host_defined, true} + ]), + {StatusCode3, ReasonPhrase3} = status(Response3), + ?assertEqual({200, "OK"}, {StatusCode3, ReasonPhrase3}), + ?assertEqual(iolist_to_binary(Body), body(Response3)) + . bad_url() -> ?assertError(_, lhttpc:request(ost, "GET", [], 100)).