Current section
Files
Jump to
Current section
Files
src/aws_request.erl
-module(aws_request).
-export([ add_headers/2
, add_query/2
, build_headers/2
, build_custom_headers/2
, method_to_binary/1
, sign_request/5
, request/2
]).
-include_lib("hackney/include/hackney_lib.hrl").
%%====================================================================
%% API
%%====================================================================
%% Perform the actual request and depending on configuration,
%% retry if the response is off a retriable type.
request(RequestFun, Options) ->
RetryState = init_retry_state(proplists:get_value(retry_options, Options, undefined)),
do_request(RequestFun, RetryState).
%% Generate headers with an AWS signature version 4 for the specified
%% request.
sign_request(Client, Method, URL, Headers, Body) ->
AccessKeyID = maps:get(access_key_id, Client),
SecretAccessKey = maps:get(secret_access_key, Client),
Region = maps:get(region, Client),
Service = maps:get(service, Client),
Token = maps:get(token, Client, undefined),
sign_request(AccessKeyID, SecretAccessKey, Region, Service, Token,
Method, URL, Headers, Body).
%% @doc Include additions only if they don't already exist in the provided list.
add_headers([], Headers) ->
Headers;
add_headers([{Name, _} = Header | Additions], Headers) ->
case lists:keyfind(Name, 1, Headers) of
false -> add_headers(Additions, [Header | Headers]);
_ -> add_headers(Additions, Headers)
end.
%% @doc Build request headers based on a list key-value pairs
%% representing the mappings from param names to header names and a
%% map with the `params'.
build_headers(ParamsHeadersMapping, Params0)
when is_list(ParamsHeadersMapping),
is_map(Params0) ->
Fun = fun({HeaderName, ParamName}, {HeadersAcc, ParamsAcc}) ->
case maps:get(ParamName, ParamsAcc, undefined) of
undefined ->
{HeadersAcc, ParamsAcc};
Value ->
Headers = [{HeaderName, Value} | HeadersAcc],
Params = maps:remove(ParamName, ParamsAcc),
{Headers, Params}
end
end,
lists:foldl(Fun, {[], Params0}, ParamsHeadersMapping).
%% @doc Build custom request headers based on a list key-value pairs
%% representing the mappings from param names to header names and a
%% map with the `params'.
build_custom_headers(ParamsCustomHeadersMapping, Params0)
when is_list(ParamsCustomHeadersMapping),
is_map(Params0) ->
Fun = fun({HeaderName, ParamName}, {HeadersAcc, ParamsAcc}) ->
case maps:get(ParamName, ParamsAcc, undefined) of
undefined ->
{HeadersAcc, ParamsAcc};
Value ->
Headers = [{<<HeaderName/binary, K/binary>>, V}
|| {K, V} <- maps:to_list(Value)] ++ HeadersAcc,
Params = maps:remove(ParamName, ParamsAcc),
{Headers, Params}
end
end,
lists:foldl(Fun, {[], Params0}, ParamsCustomHeadersMapping).
%% @doc Add querystring to url is there are any parameters in the list
-spec add_query(binary(), [{binary(), any()}]) -> binary().
add_query(Url0, Query0) ->
HackneyUrl = hackney_url:parse_url(Url0),
NewQs = iolist_to_binary(
aws_util:encode_query(
hackney_url:parse_qs(HackneyUrl#hackney_url.qs) ++ Query0)),
HackneyUrlWithAddedQs = HackneyUrl#hackney_url{qs = NewQs},
hackney_url:unparse_url(HackneyUrlWithAddedQs).
-spec method_to_binary(atom()) -> binary().
method_to_binary(delete) -> <<"DELETE">>;
method_to_binary(get) -> <<"GET">>;
method_to_binary(head) -> <<"HEAD">>;
method_to_binary(options) -> <<"OPTIONS">>;
method_to_binary(patch) -> <<"PATCH">>;
method_to_binary(post) -> <<"POST">>;
method_to_binary(put) -> <<"PUT">>.
%%====================================================================
%% Internal functions
%%====================================================================
%% Generate headers with an AWS signature version 4 for the specified
%% request.
sign_request(AccessKeyID, SecretAccessKey, Region, Service, Token,
Method, URL, Headers, Body) ->
sign_request(AccessKeyID, SecretAccessKey, Region, Service, Token,
calendar:universal_time(), Method, URL, Headers, Body).
%% Generate headers with an AWS signature version 4 for the specified
%% request using the specified time when generating signatures.
sign_request(AccessKeyID, SecretAccessKey, Region, Service, Token, Now,
Method, URL, Headers0, Body) ->
LongDate = list_to_binary(ec_date:format("YmdTHisZ", Now)),
ShortDate = list_to_binary(ec_date:format("Ymd", Now)),
Headers1 = add_date_header(Headers0, LongDate),
Headers2 = add_content_hash_header(Headers1, Body),
Headers3 = add_checksum_hash_header(Headers2, Body),
Headers = maybe_add_token_header(Headers3, Token),
CanonicalRequest = canonical_request(Method, URL, Headers, Body),
HashedCanonicalRequest = aws_util:sha256_hexdigest(CanonicalRequest),
CredentialScope = credential_scope(ShortDate, Region, Service),
SigningKey = signing_key(SecretAccessKey, ShortDate, Region, Service),
StringToSign = string_to_sign(LongDate, CredentialScope,
HashedCanonicalRequest),
Signature = aws_util:hmac_sha256_hexdigest(SigningKey, StringToSign),
SignedHeaders = signed_headers(Headers),
Authorization = authorization(AccessKeyID, CredentialScope, SignedHeaders,
Signature),
add_authorization_header(Headers, Authorization).
%% Add an Authorization header with an AWS4-HMAC-SHA256 signature to the
%% list of headers.
add_authorization_header(Headers, Authorization) ->
[{<<"Authorization">>, Authorization}|Headers].
%% Add an X-Amz-Date header with a long date value in YYMMDDTHHMMSSZ format
%% to a list of headers.
add_date_header(Headers, Date) ->
[{<<"X-Amz-Date">>, Date}|Headers].
%% Add an X-Amz-Content-SHA256 header which is the hash of the payload.
%% This header is required for S3 when using the v4 signature. Adding it
%% in requests for all services does not cause any issues.
add_content_hash_header(Headers, Body) ->
[ {<<"X-Amz-Content-SHA256">>, aws_util:sha256_hexdigest(Body)}
| Headers
].
add_checksum_hash_header(Headers, Body) ->
[ {<<"X-Amz-CheckSum-SHA256">>, base64:encode(crypto:hash(sha256, Body))}
| Headers
].
%% Add an X-Amz-Security-Token header with the user-submitted security token
%% to a list of headers
maybe_add_token_header(Headers, undefined) ->
Headers;
maybe_add_token_header(Headers, Token) ->
[{<<"X-Amz-Security-Token">>, Token}|Headers].
%% Generate an AWS4-HMAC-SHA256 authorization signature.
authorization(AccessKeyID, CredentialScope, SignedHeaders, Signature) ->
<< <<"AWS4-HMAC-SHA256 ">>/binary,
<<"Credential=">>/binary, AccessKeyID/binary,
<<"/">>/binary, CredentialScope/binary,
<<", SignedHeaders=">>/binary, SignedHeaders/binary,
<<", Signature=">>/binary, Signature/binary >>.
%% Generate a signing key from a secret access key, a short date in YYMMDD
%% format, a region identifier and a service identifier.
signing_key(SecretAccessKey, ShortDate, Region, Service) ->
SigningKey = << <<"AWS4">>/binary, SecretAccessKey/binary>>,
SignedDate = aws_util:hmac_sha256(SigningKey, ShortDate),
SignedRegion = aws_util:hmac_sha256(SignedDate, Region),
SignedService = aws_util:hmac_sha256(SignedRegion, Service),
aws_util:hmac_sha256(SignedService, <<"aws4_request">>).
%% Generate a credential scope from a short date in YYMMDD format, a
%% region identifier and a service identifier.
credential_scope(ShortDate, Region, Service) ->
aws_util:binary_join([ShortDate, Region, Service, <<"aws4_request">>],
"/").
%% Generate the text to sign from a long date in YYMMDDTHHMMSSZ format, a
%% credential scope and a hashed canonical request.
string_to_sign(LongDate, CredentialScope, HashedCanonicalRequest) ->
aws_util:binary_join([<<"AWS4-HMAC-SHA256">>, LongDate, CredentialScope,
HashedCanonicalRequest],
"\n").
%% Process and merge request values into a canonical request for AWS
%% signature version 4.
canonical_request(Method, URL, Headers, Body) ->
{CanonicalURL, CanonicalQueryString} = split_url(URL),
CanonicalHeaders = canonical_headers(Headers),
SignedHeaders = signed_headers(Headers),
PayloadHash = aws_util:sha256_hexdigest(Body),
aws_util:binary_join([Method, CanonicalURL, CanonicalQueryString,
CanonicalHeaders, SignedHeaders, PayloadHash],
<<"\n">>).
%% Strip the query string from the URL, if one if present, and return the
%% URL and query string as separate values.
split_url(URL) ->
URI = hackney_url:parse_url(URL),
{ensure_path(URI#hackney_url.path), URI#hackney_url.qs}.
%% Convert a list of headers to canonical header format. Leading and
%% trailing whitespace around header names and values is stripped, header
%% names are lowercased, and headers are newline-joined in alphabetical
%% order (with a trailing newline).
canonical_headers(Headers) ->
list_to_binary(lists:sort(lists:map(fun canonical_header/1, Headers))).
%% Strip leading and trailing whitespace around Name and Value, convert
%% Name to lowercase, and add a trailing newline.
canonical_header({Name, Value}) ->
N = list_to_binary(string:strip(string:to_lower(binary_to_list(Name)))),
V = list_to_binary(string:strip(binary_to_list(Value))),
<<N/binary, <<":">>/binary, V/binary, <<"\n">>/binary >>.
%% Convert a list of headers to canonicals signed header format. Leading
%% and trailing whitespace around names is stripped, header names are
%% lowercased, and header names are semicolon-joined in alphabetical order.
signed_headers(Headers) ->
aws_util:binary_join(lists:sort(lists:map(fun signed_header/1, Headers)),
<<";">>).
%% Strip leading and trailing whitespace around Name and convert it to
%% lowercase.
signed_header({Name, _}) ->
list_to_binary(string:strip(string:to_lower(binary_to_list(Name)))).
ensure_path(<<"">>) -> <<"/">>;
ensure_path(Path) -> Path.
do_request(RequestFun, RetryState) ->
Response = RequestFun(),
case classify_response(Response) of
retriable ->
case should_retry(RetryState) of
{ok, NewRetryState} ->
do_request(RequestFun, NewRetryState);
false ->
Response
end;
error ->
Response;
ok ->
Response
end.
init_retry_state(undefined) ->
undefined;
init_retry_state({exponential_with_jitter, {MaxAttempts, BaseSleepTime, CapSleepTime}}) ->
#{ type => exponential_with_jitter
, n => 0
, max_attempts => MaxAttempts
, base_sleep_time => BaseSleepTime
, cap_sleep_time => CapSleepTime
}.
classify_response({error, _, {StatusCode, _, _}})
when is_integer(StatusCode) andalso StatusCode >= 500 ->
retriable;
classify_response({error, {StatusCode, _}})
when is_integer(StatusCode) andalso StatusCode >= 500 ->
retriable;
classify_response({error, _, {StatusCode, _, _}})
when is_integer(StatusCode) ->
error;
classify_response({error, closed}) ->
retriable;
classify_response({error, connect_timeout}) ->
retriable;
classify_response({error, checkout_timeout}) ->
retriable;
classify_response({error, _}) ->
error;
classify_response({ok, {_, _}}) ->
ok;
classify_response({ok, _, {_, _, _}}) ->
ok.
should_retry(undefined) ->
false;
should_retry(#{ type := exponential_with_jitter
, n := N
, max_attempts := MaxAttempts}) when N =:= MaxAttempts ->
false;
should_retry(#{ type := exponential_with_jitter
, n := N
, base_sleep_time := BaseSleepTime
, cap_sleep_time := CapSleepTime} = RetryState0) ->
Temp = min(CapSleepTime, BaseSleepTime * trunc(math:pow(2, N))),
Sleep = Temp div 2 + rand:uniform(Temp div 2),
timer:sleep(Sleep),
{ok, RetryState0#{ n => N +1 }}.
%%====================================================================
%% Unit tests
%%====================================================================
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
%% sign_request/5 extracts credentials, service and region information from
%% a client map and generates an AWS signature version 4 for a request. It
%% returns a new set of HTTP headers with Authorization and X-Aws-Date
%% header/value pairs added.
sign_request_with_client_test() ->
Client = #{access_key_id => <<"access-key-id">>,
secret_access_key => <<"secret-access-key">>,
endpoint => <<"amazonaws.com">>,
region => <<"us-east-1">>,
service => <<"ec2">>},
Method = <<"GET">>,
URL = <<"https://ec2.us-east-1.amazonaws.com?Action=DescribeInstances&Version=2014-10-01">>,
Headers = [{<<"Host">>, <<"ec2.us-east-1.amazonaws.com">>},
{<<"Header">>, <<"Value">>}],
Body = <<"">>,
SignedHeaders = sign_request(Client, Method, URL, Headers, Body),
?assertEqual(true, proplists:is_defined(<<"Authorization">>, SignedHeaders)),
?assertEqual(true, proplists:is_defined(<<"X-Amz-Date">>, SignedHeaders)).
sign_request_with_temporary_client_test() ->
Client = #{access_key_id => <<"access-key-id">>,
secret_access_key => <<"secret-access-key">>,
endpoint => <<"amazonaws.com">>,
region => <<"us-east-1">>,
token => <<"my-token">>,
service => <<"ec2">>},
Method = <<"GET">>,
URL = <<"https://ec2.us-east-1.amazonaws.com?Action=DescribeInstances&Version=2014-10-01">>,
Headers = [{<<"Host">>, <<"ec2.us-east-1.amazonaws.com">>},
{<<"Header">>, <<"Value">>}],
Body = <<"">>,
SignedHeaders = sign_request(Client, Method, URL, Headers, Body),
?assertEqual(true, proplists:is_defined(<<"Authorization">>, SignedHeaders)),
?assertEqual(true, proplists:is_defined(<<"X-Amz-Security-Token">>, SignedHeaders)),
?assertEqual(true, proplists:is_defined(<<"X-Amz-Date">>, SignedHeaders)).
%% sign_request/8 generates an AWS signature version 4 for a request and
%% returns a new set of HTTP headers with Authorization and X-Aws-Date
%% header/value pairs added.
sign_request_test() ->
AccessKeyID = <<"access-key-id">>,
SecretAccessKey = <<"secret-access-key">>,
Region = <<"us-east-1">>,
Service = <<"ec2">>,
Now = {{2015, 4, 3}, {21, 31, 17}},
Method = <<"GET">>,
URL = <<"https://ec2.us-east-1.amazonaws.com?Action=DescribeInstances&Version=2014-10-01">>,
Headers = [{<<"Host">>, <<"ec2.us-east-1.amazonaws.com">>},
{<<"Header">>, <<"Value">>}],
Body = <<"">>,
?assertEqual([{<<"Authorization">>, <<"AWS4-HMAC-SHA256 Credential=access-key-id/20150403/us-east-1/ec2/aws4_request, SignedHeaders=header;host;x-amz-checksum-sha256;x-amz-content-sha256;x-amz-date, Signature=d8df9584891c2194326521b47afb1efcdd632c39222f1e3fe95f3e190e5d84b5">>},
{<<"X-Amz-CheckSum-SHA256">>, <<"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=">>},
{<<"X-Amz-Content-SHA256">>, <<"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">>},
{<<"X-Amz-Date">>, <<"20150403T213117Z">>},
{<<"Host">>, <<"ec2.us-east-1.amazonaws.com">>},
{<<"Header">>, <<"Value">>}],
sign_request(AccessKeyID, SecretAccessKey, Region, Service, undefined, Now, Method, URL, Headers, Body)).
%% add_authorization_header/2 adds an Authorization header to a list of
%% headers.
add_authorization_header_test() ->
?assertEqual([{<<"Authorization">>, <<"AWS4-HMAC-SHA256 ...">>},
{<<"Host">>, <<"example.com">>}],
add_authorization_header([{<<"Host">>, <<"example.com">>}],
<<"AWS4-HMAC-SHA256 ...">>)).
%% add_date_header/2 adds an X-Amz-Date header to a list of headers.
add_date_header_test() ->
?assertEqual([{<<"X-Amz-Date">>, <<"20150326T221217Z">>},
{<<"Host">>, <<"example.com">>}],
add_date_header([{<<"Host">>, <<"example.com">>}],
<<"20150326T221217Z">>)).
%% signing_key/4 creates a signing key from a secret access key, short
%% date, region identifier and service identifier.
signing_key_test() ->
?assertEqual(
<<108, 238, 174, 127, 62, 29, 151, 251,
60, 200, 152, 110, 95, 108, 195, 104,
208, 222, 84, 216, 129, 34, 102, 127,
208, 93, 22, 61, 71, 54, 199, 206>>,
signing_key(<<"secret-access-key">>, <<"20150326">>, <<"us-east-1">>,
<<"s3">>)).
%% credential_scope/3 combines a short date, region and service name and
%% signature identifier into a slash-joined binary value.
credential_scope_test() ->
?assertEqual(<<"20150325/us-east-1/iam/aws4_request">>,
credential_scope(<<"20150325">>, <<"us-east-1">>, <<"iam">>)).
%% string_to_sign/3 combines a long date, credential scope and hash
%% canonical request into a binary value that's ready to sign.
string_to_sign_test() ->
LongDate = <<"20150326T202136Z">>,
CredentialScope = credential_scope(
<<"20150325">>, <<"us-east-1">>, <<"iam">>),
CanonicalRequest = canonical_request(
<<"GET">>, <<"https://example.com">>,
[{<<"Host">>, <<"example.com">>},
{<<"X-Amz-Date">>, <<"20150325T105958Z">>}],
<<"">>),
HashedCanonicalRequest = aws_util:sha256_hexdigest(CanonicalRequest),
?assertEqual
(<< <<"AWS4-HMAC-SHA256">>/binary, <<"\n">>/binary,
LongDate/binary, <<"\n">>/binary,
CredentialScope/binary, <<"\n">>/binary,
HashedCanonicalRequest/binary>>,
string_to_sign(LongDate, CredentialScope, HashedCanonicalRequest)).
%% canonical_request/4 converts an HTTP method, URL, headers and body into
%% a canonical request for AWS signature version 4
canonical_request_test() ->
?assertEqual(
aws_util:binary_join(
[<<"GET">>,
<<"/">>,
<<"">>,
<<"host:example.com">>,
<<"x-amz-date:20150325T105958Z">>,
<<"">>,
<<"host;x-amz-date">>,
<<"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">>],
"\n"),
canonical_request(<<"GET">>, <<"https://example.com">>,
[{<<"Host">>, <<"example.com">>},
{<<"X-Amz-Date">>, <<"20150325T105958Z">>}],
<<"">>)).
%% split_url/1 splits a URL from its query string, URL encodes the query
%% string, and returns the URL and query string as separate values.
split_url_test() ->
?assertEqual({<<"/index">>, <<"one=1&two=2">>},
split_url(<<"https://example.com/index?one=1&two=2">>)).
%% split_url/1 returns an binary if a query string is present.
split_url_with_all_uri_elements_test() ->
?assertEqual(
{<<"/index">>, <<"one=1">>},
split_url(<<"https://username:secret@example.com:80/index?one=1">>)).
add_query_test() ->
?assertEqual(<<"https://example.com/index?one=">>,
add_query(<<"https://example.com/index?one">>, [])).
add_query_without_query_string_test() ->
?assertEqual(<<"https://example.com">>,
add_query(<<"https://example.com?">>, [])).
add_query_sorted_test() ->
?assertEqual(<<"https://example.com/index?one=1&two=2">>,
add_query(<<"https://example.com/index">>, [{<<"two">>, <<"2">>}, {<<"one">>, <<"1">>}])).
add_query_sorted_with_existing_query_test() ->
?assertEqual(<<"https://example.com/index?one=1&two=2&x=y">>,
add_query(<<"https://example.com/index?x=y">>, [{<<"two">>, <<"2">>}, {<<"one">>, <<"1">>}])).
add_query_with_fragment_test() ->
?assertEqual(<<"https://example.com/index?one=1&two=2&x=y#heyhey">>,
add_query(<<"https://example.com/index?x=y#heyhey">>,
[{<<"two">>, <<"2">>}, {<<"one">>, <<"1">>}])).
%% canonical_headers/1 returns a newline-delimited list of trimmed and
%% lowecase headers, sorted in alphabetical order, and with a trailing
%% newline.
canonical_headers_test() ->
Headers = [{<<"X-Amz-Date">>, <<"20150325T105958Z">>},
{<<"Host">>, <<"example.com">>}],
?assertEqual(<<"host:example.com\nx-amz-date:20150325T105958Z\n">>,
canonical_headers(Headers)).
%% canonical_header/1 lowercases and colon-joins a header name and value
%% and adds a trailing newline.
canonical_header_test() ->
?assertEqual(<<"host:example.com\n">>,
canonical_header({<<"Host">>, <<"example.com">>})).
%% canonical_header/1 strips leading and trailing whitespace from the
%% header name and value.
canonical_header_strips_whitespace_test() ->
?assertEqual(<<"host:example.com\n">>,
canonical_header({<<" Host ">>, <<" example.com ">>})).
%% signed_headers/1 lowercases and semicolon-joins header names in
%% alphabetic order.
signed_headers_test() ->
Headers = [{<<"X-Amz-Date">>, <<"20150325T105958Z">>},
{<<"Host">>, <<"example.com">>},
{<<"Header">>, <<"Value">>}],
?assertEqual(<<"header;host;x-amz-date">>, signed_headers(Headers)).
%% signed_header/1 lowercases the header name.
signed_header_test() ->
?assertEqual(<<"host">>, signed_header({<<"Host">>, <<"example.com">>})).
%% signed_header/1 lowercases and strips leading and trailing whitespace
%% from the header name.
signed_header_strips_whitespace_test() ->
?assertEqual(<<"host">>, signed_header({<<" Host ">>, <<"example.com">>})).
-endif.