Packages
erlcloud
0.9.1
3.8.3
3.8.2
3.8.1
3.7.6
3.7.4
3.7.3
3.7.2
3.7.1
3.7.0
3.6.8
3.6.7
3.6.5
3.6.4
3.6.3
3.6.2
3.6.1
3.6.0
3.5.16
3.5.15
3.5.14
3.5.13
3.5.12
3.5.11
3.5.10
3.5.9
3.5.8
3.5.7
3.5.6
3.5.5
3.5.4
3.5.3
3.5.2
3.5.1
3.5.0
3.4.5
3.4.3
3.4.1
3.4.0
3.3.9
3.3.8
3.3.7
3.3.6
3.3.5
3.3.4
3.3.3
3.3.2
3.3.1
3.3.0
3.2.18
3.2.17
3.2.16
3.2.15
3.2.14
3.2.13
3.2.12
3.2.11
3.2.10
3.2.7
3.2.6
3.2.5
3.2.4
3.2.3
3.2.2
3.2.1
3.2.0
3.1.17
3.1.16
3.1.14
3.1.13
3.1.12
3.1.11
3.1.9
3.1.8
3.1.7
3.1.6
3.1.5
3.1.4
3.1.3
3.1.2
3.1.1
3.1.0
3.0.5
3.0.4
3.0.3
3.0.2
3.0.1
2.2.16
2.2.15
2.2.14
2.2.13
2.2.12
2.2.11
2.2.10
2.2.9
2.2.8
2.2.7
2.2.6
2.2.5
2.2.4
2.2.2
2.2.1
2.2.0
2.1.0
2.0.5
2.0.4
2.0.3
2.0.0
0.13.10
0.13.9
0.13.8
0.13.6
0.13.5
0.13.4
0.13.3
0.13.2
0.13.0
0.12.0
0.11.0
0.9.2
0.9.2-rc.1
0.9.1
0.9.0
AWS APIs library for Erlang
Current section
Files
Jump to
Current section
Files
src/erlcloud_aws.erl
-module(erlcloud_aws).
-export([aws_request/5, aws_request/6, aws_request/7, aws_request/8,
aws_request_xml/5, aws_request_xml/6, aws_request_xml/7, aws_request_xml/8,
aws_request2/7,
aws_request_xml2/5, aws_request_xml2/7,
aws_request_form/8,
param_list/2, default_config/0, update_config/1, format_timestamp/1,
http_headers_body/1,
request_to_return/1,
sign_v4/5]).
-include("erlcloud.hrl").
-include_lib("erlcloud/include/erlcloud_aws.hrl").
-record(metadata_credentials,
{access_key_id :: string(),
secret_access_key :: string(),
security_token=undefined :: string(),
expiration_gregorian_seconds :: integer()
}).
aws_request_xml(Method, Host, Path, Params, #aws_config{} = Config) ->
Body = aws_request(Method, Host, Path, Params, Config),
element(1, xmerl_scan:string(binary_to_list(Body))).
aws_request_xml(Method, Host, Path, Params, AccessKeyID, SecretAccessKey) ->
Body = aws_request(Method, Host, Path, Params, AccessKeyID, SecretAccessKey),
element(1, xmerl_scan:string(binary_to_list(Body))).
aws_request_xml(Method, Protocol, Host, Port, Path, Params, #aws_config{} = Config) ->
Body = aws_request(Method, Protocol, Host, Port, Path, Params, Config),
element(1, xmerl_scan:string(binary_to_list(Body))).
aws_request_xml(Method, Protocol, Host, Port, Path, Params, AccessKeyID, SecretAccessKey) ->
Body = aws_request(Method, Protocol, Host, Port, Path, Params, AccessKeyID, SecretAccessKey),
element(1, xmerl_scan:string(binary_to_list(Body))).
aws_request_xml2(Method, Host, Path, Params, #aws_config{} = Config) ->
aws_request_xml2(Method, undefined, Host, undefined, Path, Params, Config).
aws_request_xml2(Method, Protocol, Host, Port, Path, Params, #aws_config{} = Config) ->
case aws_request2(Method, Protocol, Host, Port, Path, Params, Config) of
{ok, Body} ->
{ok, element(1, xmerl_scan:string(binary_to_list(Body)))};
{error, Reason} ->
{error, Reason}
end.
aws_request(Method, Host, Path, Params, #aws_config{} = Config) ->
aws_request(Method, undefined, Host, undefined, Path, Params, Config).
aws_request(Method, Host, Path, Params, AccessKeyID, SecretAccessKey) ->
aws_request(Method, undefined, Host, undefined, Path, Params, AccessKeyID, SecretAccessKey).
aws_request(Method, Protocol, Host, Port, Path, Params, #aws_config{} = Config) ->
case aws_request2(Method, Protocol, Host, Port, Path, Params, Config) of
{ok, Body} ->
Body;
{error, Reason} ->
erlang:error({aws_error, Reason})
end.
aws_request(Method, Protocol, Host, Port, Path, Params, AccessKeyID, SecretAccessKey) ->
aws_request(Method, Protocol, Host, Port, Path, Params,
#aws_config{access_key_id = AccessKeyID, secret_access_key = SecretAccessKey}).
%% aws_request2 returns {ok, Body} or {error, Reason} instead of throwing as aws_request does
%% This is the preferred pattern for new APIs
aws_request2(Method, Protocol, Host, Port, Path, Params, Config) ->
case update_config(Config) of
{ok, Config1} ->
aws_request2_no_update(Method, Protocol, Host, Port, Path, Params, Config1);
{error, Reason} ->
{error, Reason}
end.
aws_request2_no_update(Method, Protocol, Host, Port, Path, Params, #aws_config{} = Config) ->
Timestamp = format_timestamp(erlang:universaltime()),
QParams = lists:sort(
[{"Timestamp", Timestamp},
{"SignatureVersion", "2"},
{"SignatureMethod", "HmacSHA1"},
{"AWSAccessKeyId", Config#aws_config.access_key_id}|Params] ++
case Config#aws_config.security_token of
undefined -> [];
Token -> [{"SecurityToken", Token}]
end),
QueryToSign = erlcloud_http:make_query_string(QParams),
RequestToSign = [string:to_upper(atom_to_list(Method)), $\n,
string:to_lower(Host), $\n, Path, $\n, QueryToSign],
Signature = base64:encode(erlcloud_util:sha_mac(Config#aws_config.secret_access_key, RequestToSign)),
Query = [QueryToSign, "&Signature=", erlcloud_http:url_encode(Signature)],
aws_request_form(Method, Protocol, Host, Port, Path, Query, [], Config).
aws_request_form(Method, Protocol, Host, Port, Path, Form, Headers, Config) ->
case Protocol of
undefined -> UProtocol = "https://";
_ -> UProtocol = [Protocol, "://"]
end,
case Port of
undefined -> URL = [UProtocol, Host, Path];
_ -> URL = [UProtocol, Host, $:, port_to_str(Port), Path]
end,
%% Note: httpc MUST be used with {timeout, timeout()} option
%% Many timeout related failures is observed at prod env
%% when library is used in 24/7 manner
Response =
case Method of
get ->
Req = lists:flatten([URL, $?, Form]),
erlcloud_httpc:request(
Req, get, Headers, <<>>, Config#aws_config.timeout, Config);
_ ->
erlcloud_httpc:request(
lists:flatten(URL), Method,
[{<<"content-type">>, <<"application/x-www-form-urlencoded; charset=utf-8">>} | Headers],
list_to_binary(Form), Config#aws_config.timeout, Config)
end,
http_body(Response).
param_list([], _Key) -> [];
param_list(Values, Key) when is_tuple(Key) ->
Seq = lists:seq(1, size(Key)),
lists:flatten(
[[{lists:append([element(J, Key), ".", integer_to_list(I)]),
element(J, Value)} || J <- Seq] ||
{I, Value} <- lists:zip(lists:seq(1, length(Values)), Values)]
);
param_list([[{_, _}|_]|_] = Values, Key) ->
lists:flatten(
[[{lists:flatten([Key, $., integer_to_list(I), $., SubKey]),
value_to_string(Value)} || {SubKey, Value} <- SValues] ||
{I, SValues} <- lists:zip(lists:seq(1, length(Values)), Values)]
);
param_list(Values, Key) ->
[{lists:flatten([Key, $., integer_to_list(I)]), Value} ||
{I, Value} <- lists:zip(lists:seq(1, length(Values)), Values)].
value_to_string(Integer) when is_integer(Integer) -> integer_to_list(Integer);
value_to_string(Atom) when is_atom(Atom) -> atom_to_list(Atom);
value_to_string(Binary) when is_binary(Binary) -> Binary;
value_to_string(String) when is_list(String) -> String;
value_to_string({{_Yr, _Mo, _Da}, {_Hr, _Min, _Sec}} = Timestamp) -> format_timestamp(Timestamp).
format_timestamp({{Yr, Mo, Da}, {H, M, S}}) ->
lists:flatten(
io_lib:format("~4.10.0b-~2.10.0b-~2.10.0bT~2.10.0b:~2.10.0b:~2.10.0bZ",
[Yr, Mo, Da, H, M, S])).
default_config() ->
case get(aws_config) of
undefined ->
#aws_config{access_key_id=os:getenv("AWS_ACCESS_KEY_ID"),
secret_access_key=os:getenv("AWS_SECRET_ACCESS_KEY")};
Config ->
Config
end.
-spec update_config(aws_config()) -> {ok, aws_config()} | {error, term()}.
update_config(#aws_config{access_key_id = KeyId} = Config)
when is_list(KeyId) ->
%% In order to support caching of the aws_config, we could store the expiration_time
%% and check it here. If it is about to expire (within 5 minutes is what boto uses)
%% then we should get the new config.
{ok, Config};
update_config(#aws_config{} = Config) ->
%% AccessKey is not set. Try to read from role metadata.
case get_metadata_credentials(Config) of
{error, Reason} ->
{error, Reason};
{ok, Credentials} ->
{ok, Config#aws_config {
access_key_id = Credentials#metadata_credentials.access_key_id,
secret_access_key = Credentials#metadata_credentials.secret_access_key,
security_token = Credentials#metadata_credentials.security_token}}
end.
-spec get_metadata_credentials(aws_config()) -> {ok, #metadata_credentials{}} | {error, term()}.
get_metadata_credentials(Config) ->
%% See if we have cached credentials
case application:get_env(erlcloud, metadata_credentials) of
{ok, #metadata_credentials{expiration_gregorian_seconds = Expiration} = Credentials} ->
Now = calendar:datetime_to_gregorian_seconds(calendar:universal_time()),
%% Get new credentials if these will expire in less than 5 minutes
case Expiration - Now < 300 of
true -> get_credentials_from_metadata(Config);
false -> {ok, Credentials}
end;
undefined ->
get_credentials_from_metadata(Config)
end.
timestamp_to_gregorian_seconds(Timestamp) ->
{ok, [Yr, Mo, Da, H, M, S], []} = io_lib:fread("~d-~d-~dT~d:~d:~dZ", binary_to_list(Timestamp)),
calendar:datetime_to_gregorian_seconds({{Yr, Mo, Da}, {H, M, S}}).
-spec get_credentials_from_metadata(aws_config())
-> {ok, #metadata_credentials{}} | {error, term()}.
get_credentials_from_metadata(Config) ->
%% TODO this function should retry on errors getting credentials
%% First get the list of roles
case http_body(
erlcloud_httpc:request(
"http://169.254.169.254/latest/meta-data/iam/security-credentials/",
get, [], <<>>, Config#aws_config.timeout, Config)) of
{error, Reason} ->
{error, Reason};
{ok, Body} ->
%% Always use the first role
[Role | _] = binary:split(Body, <<$\n>>),
case http_body(
erlcloud_httpc:request(
"http://169.254.169.254/latest/meta-data/iam/security-credentials/" ++
binary_to_list(Role),
get, [], <<>>, Config#aws_config.timeout, Config)) of
{error, Reason} ->
{error, Reason};
{ok, Json} ->
Creds = jsx:decode(Json),
Record = #metadata_credentials
{access_key_id = binary_to_list(proplists:get_value(<<"AccessKeyId">>, Creds)),
secret_access_key = binary_to_list(proplists:get_value(<<"SecretAccessKey">>, Creds)),
security_token = binary_to_list(proplists:get_value(<<"Token">>, Creds)),
expiration_gregorian_seconds = timestamp_to_gregorian_seconds(
proplists:get_value(<<"Expiration">>, Creds))},
application:set_env(erlcloud, metadata_credentials, Record),
{ok, Record}
end
end.
port_to_str(Port) when is_integer(Port) ->
integer_to_list(Port);
port_to_str(Port) when is_list(Port) ->
Port.
-spec http_body({ok, tuple()} | {error, term()})
-> {ok, string() | binary()} | {error, tuple()}.
%% Extract the body and do error handling on the return of a httpc:request call.
http_body(Return) ->
case http_headers_body(Return) of
{ok, {_, Body}} ->
{ok, Body};
{error, Reason} ->
{error, Reason}
end.
-type headers() :: [{string(), string()}].
-spec http_headers_body({ok, tuple()} | {error, term()})
-> {ok, {headers(), string() | binary()}} | {error, tuple()}.
%% Extract the headers and body and do error handling on the return of a httpc:request call.
http_headers_body({ok, {{OKStatus, _StatusLine}, Headers, Body}})
when OKStatus >= 200, OKStatus =< 299 ->
{ok, {Headers, Body}};
http_headers_body({ok, {{Status, StatusLine}, _Headers, Body}}) ->
{error, {http_error, Status, StatusLine, Body}};
http_headers_body({error, Reason}) ->
{error, {socket_error, Reason}}.
%% Convert an aws_request record to return value as returned by http_headers_body
request_to_return(#aws_request{response_type = ok,
response_headers = Headers,
response_body = Body}) ->
{ok, {Headers, Body}};
request_to_return(#aws_request{response_type = error,
error_type = httpc,
httpc_error_reason = Reason}) ->
{error, {socket_error, Reason}};
request_to_return(#aws_request{response_type = error,
error_type = aws,
response_status = Status,
response_status_line = StatusLine,
response_body = Body}) ->
{error, {http_error, Status, StatusLine, Body}}.
%% http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
%% TODO additional parameters - currently only supports what is needed for DynamoDB
-spec sign_v4(aws_config(), headers(), binary(), string(), string()) -> headers().
sign_v4(Config, Headers, Payload, Region, Service) ->
Date = iso_8601_basic_time(),
Headers1 = [{"x-amz-date", Date} | Headers],
Headers2 = case Config#aws_config.security_token of
undefined -> Headers1;
Token -> [{"x-amz-security-token", Token} | Headers1]
end,
{Request, SignedHeaders} = canonical_request("POST", "/", "", Headers2, Payload),
CredentialScope = credential_scope(Date, Region, Service),
ToSign = to_sign(Date, CredentialScope, Request),
SigningKey = signing_key(Config, Date, Region, Service),
Signature = base16(erlcloud_util:sha256_mac( SigningKey, ToSign)),
Authorization = authorization(Config, CredentialScope, SignedHeaders, Signature),
[{"Authorization", lists:flatten(Authorization)} | Headers2].
iso_8601_basic_time() ->
{{Year,Month,Day},{Hour,Min,Sec}} = calendar:now_to_universal_time(os:timestamp()),
lists:flatten(io_lib:format(
"~4.10.0B~2.10.0B~2.10.0BT~2.10.0B~2.10.0B~2.10.0BZ",
[Year, Month, Day, Hour, Min, Sec])).
canonical_request(Method, CanonicalURI, CanonicalQueryString, Headers, Payload) ->
{CanonicalHeaders, SignedHeaders} = canonical_headers(Headers),
{[Method, $\n,
CanonicalURI, $\n,
CanonicalQueryString, $\n,
CanonicalHeaders, $\n,
SignedHeaders, $\n,
hash_encode(Payload)],
SignedHeaders}.
canonical_headers(Headers) ->
Normalized = [{string:to_lower(Name), trimall(Value)} || {Name, Value} <- Headers],
Sorted = lists:keysort(1, Normalized),
Canonical = [[Name, $:, Value, $\n] || {Name, Value} <- Sorted],
Signed = string:join([Name || {Name, _} <- Sorted], ";"),
{Canonical, Signed}.
trimall(Value) ->
%% TODO - remove excess internal whitespace in header values
re:replace(Value, "(^\\s+)|(\\s+$)", "", [global]).
hash_encode(Data) ->
Hash = erlcloud_util:sha256( Data),
base16(Hash).
base16(Data) ->
io_lib:format("~64.16.0b", [binary:decode_unsigned(Data)]).
credential_scope(Date, Region, Service) ->
DateOnly = string:left(Date, 8),
[DateOnly, $/, Region, $/, Service, "/aws4_request"].
to_sign(Date, CredentialScope, Request) ->
["AWS4-HMAC-SHA256\n",
Date, $\n,
CredentialScope, $\n,
hash_encode(Request)].
signing_key(Config, Date, Region, Service) ->
%% TODO cache the signing key so we don't have to recompute for every request
DateOnly = string:left(Date, 8),
KDate = erlcloud_util:sha256_mac( "AWS4" ++ Config#aws_config.secret_access_key, DateOnly),
KRegion = erlcloud_util:sha256_mac( KDate, Region),
KService = erlcloud_util:sha256_mac( KRegion, Service),
erlcloud_util:sha256_mac( KService, "aws4_request").
authorization(Config, CredentialScope, SignedHeaders, Signature) ->
["AWS4-HMAC-SHA256"
" Credential=", Config#aws_config.access_key_id, $/, CredentialScope, $,,
" SignedHeaders=", SignedHeaders, $,,
" Signature=", Signature].