Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion include/lhttpc_types.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@
{send_retry, non_neg_integer()} |
{stream_to, pid()} |
{partial_upload, non_neg_integer() | infinity} |
{partial_download, pid(), non_neg_integer() | infinity}.
{partial_download, pid(), non_neg_integer() | infinity} |
{is_host_defined, boolean()} |
{is_content_length_defined, boolean()}.

-type options() :: [option()].

Expand Down
53 changes: 49 additions & 4 deletions src/lhttpc.erl
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@
%%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%%% ----------------------------------------------------------------------------

%%% @author Oscar Hellstr�m <[email protected]>
%%% @author Oscar Hellström <[email protected]>
%%% @doc Main interface to the lightweight http client.
%%% See {@link request/4}, {@link request/5} and {@link request/6} functions.
%%% @end
-module(lhttpc).
-behaviour(application).

-export([start/0, stop/0, request/4, request/5, request/6, request/9]).
-export([start/0, stop/0, request/4, request/5, request/6, request/9, simple_request/8]).
-export([start/2, stop/1]).
-export([
send_body_part/2,
Expand All @@ -47,7 +47,7 @@
-include("lhttpc_types.hrl").

-type result() :: {ok, {{pos_integer(), string()}, headers(), binary()}} |
{error, atom()}.
{error, term()}.

%% @hidden
-spec start(normal | {takeover, node()} | {failover, node()}, any()) ->
Expand Down Expand Up @@ -334,12 +334,17 @@ request(URL, Method, Hdrs, Body, Timeout, Options) ->
%% lhttpc will never close the connection itself; it will only be
%% closed by the other end or the TCP keepalive mechanism.
%%
%% `{is_content_length_defined, Bool}' and `{is_host_defined, Bool}'
%% is used to skip lhttpc_lib:header_value/2,3.
%% When boolean value is specified, there is no need to check if the
%% value exists.
%%
%% @end
-spec request(string(), 1..65535, true | false, string(), atom() | string(),
headers(), iolist(), pos_integer(), [option()]) -> result().
request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) ->
verify_options(Options, []),
ReqId = now(),
ReqId = erlang:unique_integer(),
case proplists:is_defined(stream_to, Options) of
true ->
StreamTo = proplists:get_value(stream_to, Options),
Expand All @@ -355,6 +360,8 @@ request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) ->
Pid = spawn_link(lhttpc_client, request, Args),
receive
{response, ReqId, Pid, R} ->
%% all throw/1 and 'connection_closed', in addition to normal
%% case is mapped here
R;
{exit, ReqId, Pid, Reason} ->
% We would rather want to exit here, instead of letting the
Expand All @@ -371,6 +378,40 @@ request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) ->
end
end.

%% @spec (Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> Result
%% Host = string()
%% Port = 1..65535
%% Ssl = boolean()
%% Path = string()
%% Method = atom() | string()
%% Hdrs = headers()
%% Body = iolist()
%% Options = [option()]
%% Result = result()
%% @doc This interface does not support timeout, partial upload, and
%% stream_to options. The check for `partial_upload' or `stream_to'
%% is not performed, so users have to be aware of it.
%% Host, Port, Ssl, and Path can be obtained using `lhttpc_lib:parse_url/1'.
%% @end
-spec simple_request(string(), 1..65535, true | false, string(), atom() | string(),
headers(), iolist(), [option()]) -> result().
simple_request(Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
verify_options(Options, []),
ReqId = undefined, % ReqId is unnecessary here.
Self = undefined, % self() is unnecessary here.
RetVal =
lhttpc_client:get_request_result(
ReqId, Self, Host, Port, Ssl, Path, Method, Hdrs, Body, Options
),
case RetVal of
{response, _ReqId, _Self, Response} ->
%% success + throw/1 + 'connection_closed'
Response;
{exit, _ReqId, _Self, Reason} ->
%% error/1 runtime cases
exit(Reason)
end.

%% @spec (UploadState :: UploadState, BodyPart :: BodyPart) -> Result
%% BodyPart = iolist() | binary()
%% Timeout = integer() | infinity
Expand Down Expand Up @@ -597,6 +638,10 @@ verify_options([{connect_options, List} | Options], Errors)
verify_options(Options, Errors);
verify_options([{stream_to, Pid} | Options], Errors) when is_pid(Pid) ->
verify_options(Options, Errors);
verify_options([{is_content_length_defined, Bool} | Options], Errors) when is_boolean(Bool) ->
verify_options(Options, Errors);
verify_options([{is_host_defined, Bool} | Options], Errors) when is_boolean(Bool) ->
verify_options(Options, Errors);
verify_options([Option | Options], Errors) ->
verify_options(Options, [Option | Errors]);
verify_options([], Errors) ->
Expand Down
33 changes: 22 additions & 11 deletions src/lhttpc_client.erl
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
-module(lhttpc_client).

-export([request/10]).
-export([get_request_result/10]).

-include("lhttpc_types.hrl").

Expand Down Expand Up @@ -77,16 +78,7 @@
%% Option = {connect_timeout, Milliseconds}
%% @end
request(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
Result = try
execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options)
catch
Reason ->
{response, ReqId, self(), {error, Reason}};
error:closed ->
{response, ReqId, self(), {error, connection_closed}};
error:Error ->
{exit, ReqId, self(), {Error, erlang:get_stacktrace()}}
end,
Result = get_request_result(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options),
case Result of
{response, _, _, {ok, {no_return, _}}} -> ok;
_Else -> From ! Result
Expand All @@ -96,6 +88,21 @@ request(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
unlink(From),
ok.

get_request_result(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
try
execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options)
catch
%% all throw/1
Reason ->
{response, ReqId, self(), {error, Reason}};
%% 'error:closed' only
error:closed ->
{response, ReqId, self(), {error, connection_closed}};
%% all other error/1 runtime errors
error:Error ->
{exit, ReqId, self(), {Error, erlang:get_stacktrace()}}
end.

execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
UploadWindowSize = proplists:get_value(partial_upload, Options),
PartialUpload = proplists:is_defined(partial_upload, Options),
Expand All @@ -104,8 +111,11 @@ execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
NormalizedMethod = lhttpc_lib:normalize_method(Method),
MaxConnections = proplists:get_value(max_connections, Options, 10),
ConnectionTimeout = proplists:get_value(connection_timeout, Options, infinity),
IsContentLengthDefined = proplists:get_value(is_content_length_defined, Options, undefined),
IsHostDefined = proplists:get_value(is_host_defined, Options, undefined),

{ChunkedUpload, Request} = lhttpc_lib:format_request(Path, NormalizedMethod,
Hdrs, Host, Port, Body, PartialUpload),
Hdrs, Host, Port, Body, PartialUpload, IsContentLengthDefined, IsHostDefined),
Socket = case lhttpc_lb:checkout(Host, Port, Ssl, MaxConnections, ConnectionTimeout) of
{ok, S} -> S; % Re-using HTTP/1.1 connections
retry_later -> throw(retry_later);
Expand Down Expand Up @@ -336,6 +346,7 @@ has_body(_, 304, _) ->
has_body(_, _, _) ->
true. % All other responses are assumed to have a body

-spec body_type(Hdrs::list({string(), term()})) -> chunked | infinite | {fixed_length, integer()}.
body_type(Hdrs) ->
% Find out how to read the entity body from the request.
% * If we have a Content-Length, just use that and read the complete
Expand Down
148 changes: 98 additions & 50 deletions src/lhttpc_lib.erl
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

-export([
parse_url/1,
format_request/7,
format_request/9,
header_value/2,
header_value/3,
normalize_method/1
Expand All @@ -42,6 +42,36 @@
-export([format_hdrs/1, dec/1]).

-include("lhttpc_types.hrl").
-define(L(C),
case C of
$A -> $a;
$B -> $b;
$C -> $c;
$D -> $d;
$E -> $e;
$F -> $f;
$G -> $g;
$H -> $h;
$I -> $i;
$J -> $j;
$K -> $k;
$L -> $l;
$M -> $m;
$N -> $n;
$O -> $o;
$P -> $p;
$Q -> $q;
$R -> $r;
$S -> $s;
$T -> $t;
$U -> $u;
$V -> $v;
$W -> $w;
$X -> $x;
$Y -> $y;
$Z -> $z;
C -> C
end).

%% @spec header_value(Header, Headers) -> undefined | term()
%% Header = string()
Expand All @@ -68,14 +98,23 @@ header_value(Hdr, Hdrs) ->
%% @end
-spec header_value(string(), [{string(), Value}], Default) ->
Default | Value.
header_value(Hdr, [{Hdr, Value} | _], _) ->
header_value(Hdr, Hdrs, Default) ->
header_value_search(Hdr, Hdr, Hdrs, Default).

header_value_search(Hdr, Hdr, [{Hdr, Value} | _], _) ->
Value;
header_value_search([], _Hdr, [{[], Value}| _Hdrs], _Default) ->
Value;
header_value(Hdr, [{ThisHdr, Value}| Hdrs], Default) ->
case string:equal(string:to_lower(ThisHdr), Hdr) of
true -> Value;
false -> header_value(Hdr, Hdrs, Default)
header_value_search(_, Hdr, [{[], _}| Hdrs], Default) ->
header_value_search(Hdr, Hdr, Hdrs, Default);
header_value_search([], Hdr, [_| Hdrs], Default) ->
header_value_search(Hdr, Hdr, Hdrs, Default);
header_value_search([H0|Hdr0], Hdr, [{[H1|Hdr1], Value}| Hdrs], Default) ->
case H0 == ?L(H1) of
true -> header_value_search(Hdr0, Hdr, [{Hdr1, Value}| Hdrs], Default);
false -> header_value_search(Hdr, Hdr, Hdrs, Default)
end;
header_value(_, [], Default) ->
header_value_search(_, _Hdr, [], Default) ->
Default.

%% @spec (Item) -> OtherItem
Expand Down Expand Up @@ -151,11 +190,20 @@ split_port(Scheme, [P | T], Port) ->
%% Port = integer()
%% Body = iolist()
%% PartialUpload = true | false
%% IsContentLengthUnDefined = true | false
-spec format_request(iolist(), atom() | string(), headers(), string(),
integer(), iolist(), true | false ) -> {true | false, iolist()}.
format_request(Path, Method, Hdrs, Host, Port, Body, PartialUpload) ->
AllHdrs = add_mandatory_hdrs(Method, Hdrs, Host, Port, Body, PartialUpload),
IsChunked = is_chunked(AllHdrs),
integer(), iolist(), true | false, boolean() | 'undefined', boolean() | 'undefined') -> {true | false, iolist()}.
format_request(Path, Method, Hdrs, Host, Port, Body, PartialUpload, IsContentLengthDefined, IsHostDefined) ->
{IsChunked, ContentHdrs} =
if
Method =/= "POST" andalso Method =/= "PUT" -> {false, Hdrs};
PartialUpload == false ->
{false, add_content_length(Hdrs, Body, IsContentLengthDefined)};
PartialUpload == true ->
add_transfer_encoding(Hdrs)
end,

AllHdrs = add_host(ContentHdrs, Host, Port, IsHostDefined),
{
IsChunked,
[
Expand Down Expand Up @@ -203,60 +251,60 @@ format_body(Body, true) ->
]
end.

add_mandatory_hdrs(Method, Hdrs, Host, Port, Body, PartialUpload) ->
ContentHdrs = add_content_headers(Method, Hdrs, Body, PartialUpload),
add_host(ContentHdrs, Host, Port).
-spec add_content_length(Hdrs::headers(), Body::iolist(), IsContentLengthDefined::boolean()) -> headers().
add_content_length(Hdrs, _Body, true) -> Hdrs;
add_content_length(Hdrs, Body, false) -> add_content_length(Hdrs, Body);
add_content_length(Hdrs, Body, undefined) ->
case header_value("content-length", Hdrs) of
undefined -> add_content_length(Hdrs, Body);
_ -> Hdrs % We have a content length
end.

add_content_headers("POST", Hdrs, Body, PartialUpload) ->
add_content_headers(Hdrs, Body, PartialUpload);
add_content_headers("PUT", Hdrs, Body, PartialUpload) ->
add_content_headers(Hdrs, Body, PartialUpload);
add_content_headers(_, Hdrs, _, _PartialUpload) ->
Hdrs.
add_content_length(Hdrs, Body) ->
ContentLength = integer_to_list(iolist_size(Body)),
[{"Content-Length", ContentLength} | Hdrs].

add_content_headers(Hdrs, Body, false) ->
case header_value("content-length", Hdrs) of
undefined ->
ContentLength = integer_to_list(iolist_size(Body)),
[{"Content-Length", ContentLength} | Hdrs];
_ -> % We have a content length
Hdrs
end;
add_content_headers(Hdrs, _Body, true) ->
case {header_value("content-length", Hdrs),
header_value("transfer-encoding", Hdrs)} of
{undefined, undefined} ->
[{"Transfer-Encoding", "chunked"} | Hdrs];
{undefined, TransferEncoding} ->
case string:to_lower(TransferEncoding) of
"chunked" -> Hdrs;
_ -> erlang:error({error, unsupported_transfer_encoding})
end;
{_Length, undefined} ->
Hdrs;
{_Length, _TransferEncoding} -> %% have both cont.length and chunked

-define(CHUNKED, true).
-define(NOT_CHUNKED, false).
-spec add_transfer_encoding(Hdrs::headers()) -> {IsChunked::boolean(), Hdrs::headers()}.
add_transfer_encoding(Hdrs) ->
case {header_value("content-length", Hdrs), header_value("transfer-encoding", Hdrs)} of
{undefined, undefined} -> {?CHUNKED, [{"Transfer-Encoding", "chunked"} | Hdrs]};
{undefined, TransferEncoding} -> {?CHUNKED, confirm_chunked(TransferEncoding, Hdrs)};
{_Length, undefined} -> {?NOT_CHUNKED, Hdrs};
_Else ->
%% Have both cont.length and chunked. This can happen
%% regardless of `is_content_length_defined' flag
erlang:error({error, bad_header})
end.

add_host(Hdrs, Host, Port) ->
-spec confirm_chunked(TransferEncoding::string(), Hdrs::headers()) -> Hdrs::headers().
confirm_chunked(TransferEncoding, Hdrs) ->
case string:to_lower(TransferEncoding) of
"chunked" -> Hdrs;
_ -> erlang:error({error, unsupported_transfer_encoding})
end.


-spec add_host(Hdrs::headers(), Host::host(), Port::non_neg_integer(), IsHostDefined::boolean()) ->
Hdrs::headers().
add_host(Hdrs, _Host, _Port, true) -> Hdrs;
add_host(Hdrs, Host, Port, false) ->
[{"Host", host(Host, Port) } | Hdrs];
add_host(Hdrs, Host, Port, 'undefined') ->
case header_value("host", Hdrs) of
undefined ->
[{"Host", host(Host, Port) } | Hdrs];
_ -> % We have a host
Hdrs
end.

is_chunked(Hdrs) ->
TransferEncoding = string:to_lower(
header_value("transfer-encoding", Hdrs, "undefined")),
case TransferEncoding of
"chunked" -> true;
_ -> false
end.

-spec dec(timeout()) -> timeout().
dec(Num) when is_integer(Num) -> Num - 1;
dec(Else) -> Else.

-spec host(Host::string(), Port::port()) -> Host::string() | list().
host(Host, 80) -> Host;
host(Host, Port) -> [Host, $:, integer_to_list(Port)].

Loading