Current section

Files

Jump to
req lib req utils.ex
Raw

lib/req/utils.ex

defmodule Req.Utils do
@moduledoc false
defmacrop iodata({:<<>>, _, parts}) do
Enum.map(parts, &to_iodata/1)
end
defp to_iodata(binary) when is_binary(binary) do
binary
end
defp to_iodata(
{:"::", _, [{{:., _, [Kernel, :to_string]}, _, [interpolation]}, {:binary, _, nil}]}
) do
interpolation
end
@doc """
Create AWS Signature v4.
https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
"""
def aws_sigv4_headers(options) do
{access_key_id, options} = Keyword.pop!(options, :access_key_id)
{secret_access_key, options} = Keyword.pop!(options, :secret_access_key)
{security_token, options} = Keyword.pop(options, :token)
{region, options} = Keyword.pop!(options, :region)
{service, options} = Keyword.pop!(options, :service)
{datetime, options} = Keyword.pop!(options, :datetime)
{method, options} = Keyword.pop!(options, :method)
{url, options} = Keyword.pop!(options, :url)
{headers, options} = Keyword.pop!(options, :headers)
{body, options} = Keyword.pop!(options, :body)
Keyword.validate!(options, [:body_digest])
datetime = DateTime.truncate(datetime, :second)
datetime_string = DateTime.to_iso8601(datetime, :basic)
date_string = Date.to_iso8601(datetime, :basic)
url = normalize_url(url)
body_digest = options[:body_digest] || hex(sha256(body))
service = to_string(service)
method = method |> Atom.to_string() |> String.upcase()
headers = canonical_host_header(headers, url)
aws_headers = [
{"x-amz-content-sha256", body_digest},
{"x-amz-date", datetime_string}
]
aws_headers =
if security_token do
aws_headers ++ [{"x-amz-security-token", security_token}]
else
aws_headers
end
canonical_headers = headers ++ aws_headers
## canonical_headers needs to be sorted for canonical_request construction
canonical_headers = Enum.sort(canonical_headers)
signed_headers =
Enum.map_intersperse(
Enum.sort(canonical_headers),
";",
&String.downcase(elem(&1, 0), :ascii)
)
canonical_headers =
Enum.map_intersperse(canonical_headers, "\n", fn {name, value} -> [name, ":", value] end)
path = URI.encode(url.path || "/", &(&1 == ?/ or URI.char_unreserved?(&1)))
canonical_query = canonical_query(url.query)
canonical_request = """
#{method}
#{path}
#{canonical_query}
#{canonical_headers}
#{signed_headers}
#{body_digest}\
"""
string_to_sign =
iodata("""
AWS4-HMAC-SHA256
#{datetime_string}
#{date_string}/#{region}/#{service}/aws4_request
#{hex(sha256(canonical_request))}\
""")
signature =
aws_sigv4(
string_to_sign,
date_string,
region,
service,
secret_access_key
)
credential = "#{access_key_id}/#{date_string}/#{region}/#{service}/aws4_request"
authorization =
"AWS4-HMAC-SHA256 Credential=#{credential},SignedHeaders=#{signed_headers},Signature=#{signature}"
[{"authorization", authorization}] ++ aws_headers ++ headers
end
defp canonical_query(query) when query in [nil, ""] do
query
end
defp canonical_query(query) do
for item <- String.split(query, "&", trim: true) do
case String.split(item, "=") do
[name, value] -> [name, "=", value]
[name] -> [name, "="]
end
end
|> Enum.sort()
|> Enum.intersperse("&")
end
@doc """
Create AWS Signature v4 URL.
https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
"""
def aws_sigv4_url(options) do
{access_key_id, options} = Keyword.pop!(options, :access_key_id)
{secret_access_key, options} = Keyword.pop!(options, :secret_access_key)
{region, options} = Keyword.pop!(options, :region)
{service, options} = Keyword.pop!(options, :service)
{datetime, options} = Keyword.pop!(options, :datetime)
{method, options} = Keyword.pop!(options, :method)
{url, options} = Keyword.pop!(options, :url)
{expires, options} = Keyword.pop(options, :expires, 86400)
{headers, options} = Keyword.pop(options, :headers, [])
{query, options} = Keyword.pop(options, :query, [])
[] = options
datetime = DateTime.truncate(datetime, :second)
datetime_string = DateTime.to_iso8601(datetime, :basic)
date_string = Date.to_iso8601(datetime, :basic)
url = normalize_url(url)
service = to_string(service)
canonical_headers =
headers
|> canonical_host_header(url)
|> format_canonical_headers()
signed_headers = Enum.map_join(canonical_headers, ";", &elem(&1, 0))
canonical_query_string =
format_canonical_query_params(
[
{"X-Amz-Algorithm", "AWS4-HMAC-SHA256"},
{"X-Amz-Credential",
"#{access_key_id}/#{date_string}/#{region}/#{service}/aws4_request"},
{"X-Amz-Date", datetime_string},
{"X-Amz-Expires", expires},
{"X-Amz-SignedHeaders", signed_headers}
] ++ query
)
path = URI.encode(url.path || "/", &(&1 == ?/ or URI.char_unreserved?(&1)))
true = url.query in [nil, ""]
method = method |> Atom.to_string() |> String.upcase()
canonical_headers =
Enum.map_intersperse(canonical_headers, "\n", fn {name, value} -> [name, ":", value] end)
canonical_request = """
#{method}
#{path}
#{canonical_query_string}
#{canonical_headers}
#{signed_headers}
UNSIGNED-PAYLOAD\
"""
string_to_sign =
iodata("""
AWS4-HMAC-SHA256
#{datetime_string}
#{date_string}/#{region}/#{service}/aws4_request
#{hex(sha256(canonical_request))}\
""")
signature =
aws_sigv4(
string_to_sign,
date_string,
region,
service,
secret_access_key
)
%{url | path: path, query: canonical_query_string <> "&X-Amz-Signature=#{signature}"}
end
# Try decoding the path in case it was encoded earlier to prevent double encoding,
# as the path is encoded later in the corresponding function.
defp normalize_url(url) do
url = URI.parse(url)
case url.path do
nil -> url
path -> %{url | path: URI.decode(path)}
end
end
defp canonical_host_header(headers, %URI{} = url) do
{_host_headers, headers} = Enum.split_with(headers, &match?({"host", _value}, &1))
host_value =
if is_nil(url.port) or URI.default_port(url.scheme) == url.port do
url.host
else
"#{url.host}:#{url.port}"
end
[{"host", host_value} | headers]
end
# Headers must be sorted alphabetically by name
# Header names must be lower case
# Header values must be trimmed
# See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
defp format_canonical_headers(headers) do
headers
|> Enum.map(&format_canonical_header/1)
|> Enum.sort(fn {name_1, _}, {name_2, _} -> name_1 < name_2 end)
end
defp format_canonical_header({name, value}) do
name =
name
|> to_string()
|> String.downcase(:ascii)
value =
value
|> to_string()
|> String.trim()
{name, value}
end
# Query params must be sorted alphabetically by name
# Query param name and values must be URI-encoded individually
# Query params must be sorted after encoding
# See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
defp format_canonical_query_params(query_params) do
query_params
|> Enum.map(&format_canonical_query_param/1)
|> Enum.sort(&canonical_query_param_sorter/2)
|> Enum.map_join("&", fn {name, value} -> "#{name}=#{value}" end)
end
# Spaces must be encoded as %20, not as "+".
defp format_canonical_query_param({name, value}) do
name =
name
|> to_string()
|> URI.encode(&URI.char_unreserved?/1)
value =
value
|> to_string()
|> URI.encode(&URI.char_unreserved?/1)
{name, value}
end
defp canonical_query_param_sorter({name, value_1}, {name, value_2}), do: value_1 < value_2
defp canonical_query_param_sorter({name_1, _}, {name_2, _}), do: name_1 < name_2
def aws_sigv4(
string_to_sign,
date_string,
region,
service,
secret_access_key
) do
signature =
["AWS4", secret_access_key]
|> hmac(date_string)
|> hmac(region)
|> hmac(service)
|> hmac("aws4_request")
|> hmac(string_to_sign)
|> hex()
signature
end
defp hex(data) do
Base.encode16(data, case: :lower)
end
defp sha256(data) do
:crypto.hash(:sha256, data)
end
defp hmac(key, data) do
:crypto.mac(:hmac, :sha256, key, data)
end
@doc """
Formats a datetime as "HTTP Date".
## Examples
iex> Req.Utils.format_http_date(~U[2024-01-01 09:00:00Z])
"Mon, 01 Jan 2024 09:00:00 GMT"
"""
def format_http_date(datetime) do
Calendar.strftime(datetime, "%a, %d %b %Y %H:%M:%S GMT")
end
@doc """
Parses "HTTP Date" as datetime.
## Examples
iex> Req.Utils.parse_http_date("Mon, 01 Jan 2024 09:00:00 GMT")
{:ok, ~U[2024-01-01 09:00:00Z]}
"""
def parse_http_date(<<
day_name::binary-size(3),
", ",
day::binary-size(2),
" ",
month_name::binary-size(3),
" ",
year::binary-size(4),
" ",
time::binary-size(8),
" GMT"
>>) do
with {:ok, day_of_week} <- parse_day_name(day_name),
{day, ""} <- Integer.parse(day),
{:ok, month} <- parse_month_name(month_name),
{year, ""} <- Integer.parse(year),
{:ok, time} <- Time.from_iso8601(time),
{:ok, date} <- Date.new(year, month, day),
true <- day_of_week == Date.day_of_week(date) do
DateTime.new(date, time)
else
{:error, _} = e ->
e
_ ->
{:error, :invalid_format}
end
end
def parse_http_date(binary) when is_binary(binary) do
{:error, :invalid_format}
end
defp parse_month_name("Jan"), do: {:ok, 1}
defp parse_month_name("Feb"), do: {:ok, 2}
defp parse_month_name("Mar"), do: {:ok, 3}
defp parse_month_name("Apr"), do: {:ok, 4}
defp parse_month_name("May"), do: {:ok, 5}
defp parse_month_name("Jun"), do: {:ok, 6}
defp parse_month_name("Jul"), do: {:ok, 7}
defp parse_month_name("Aug"), do: {:ok, 8}
defp parse_month_name("Sep"), do: {:ok, 9}
defp parse_month_name("Oct"), do: {:ok, 10}
defp parse_month_name("Nov"), do: {:ok, 11}
defp parse_month_name("Dec"), do: {:ok, 12}
defp parse_month_name(_), do: :error
defp parse_day_name("Mon"), do: {:ok, 1}
defp parse_day_name("Tue"), do: {:ok, 2}
defp parse_day_name("Wed"), do: {:ok, 3}
defp parse_day_name("Thu"), do: {:ok, 4}
defp parse_day_name("Fri"), do: {:ok, 5}
defp parse_day_name("Sat"), do: {:ok, 6}
defp parse_day_name("Sun"), do: {:ok, 7}
defp parse_day_name(_), do: :error
@doc """
Parses "HTTP Date" as datetime or raises an error.
## Examples
iex> Req.Utils.parse_http_date!("Mon, 01 Jan 2024 09:00:00 GMT")
~U[2024-01-01 09:00:00Z]
iex> Req.Utils.parse_http_date!("Mon")
** (ArgumentError) cannot parse "Mon" as HTTP date, reason: :invalid_format
"""
def parse_http_date!(binary) do
case parse_http_date(binary) do
{:ok, datetime} ->
datetime
{:error, reason} ->
raise ArgumentError,
"cannot parse #{inspect(binary)} as HTTP date, reason: #{inspect(reason)}"
end
end
@doc """
Returns a stream where each element is gzipped.
## Examples
iex> gzipped = Req.Utils.stream_gzip(~w[foo bar baz]) |> Enum.to_list()
iex> :zlib.gunzip(gzipped)
"foobarbaz"
"""
def stream_gzip(enumerable) do
eof = make_ref()
enumerable
|> Stream.concat([eof])
|> Stream.transform(
fn ->
z = :zlib.open()
# https://github.com/erlang/otp/blob/OTP-26.0/erts/preloaded/src/zlib.erl#L551
:ok = :zlib.deflateInit(z, :default, :deflated, 16 + 15, 8, :default)
z
end,
fn
^eof, z ->
buf = :zlib.deflate(z, [], :finish)
{buf, z}
data, z ->
buf = :zlib.deflate(z, data)
{buf, z}
end,
fn z ->
:ok = :zlib.deflateEnd(z)
:ok = :zlib.close(z)
end
)
end
defmodule CollectWithHash do
@moduledoc false
defstruct [:collectable, :type]
defimpl Collectable do
def into(%{collectable: collectable, type: type}) do
{acc, collector} = Collectable.into(collectable)
new_collector = fn
{acc, hash}, {:cont, element} ->
hash = :crypto.hash_update(hash, element)
{collector.(acc, {:cont, element}), hash}
{acc, hash}, :done ->
hash = :crypto.hash_final(hash)
{collector.(acc, :done), hash}
{acc, hash}, :halt ->
{collector.(acc, :halt), hash}
end
hash = hash_init(type)
{{acc, hash}, new_collector}
end
defp hash_init(:sha1), do: :crypto.hash_init(:sha)
defp hash_init(type), do: :crypto.hash_init(type)
end
end
@doc """
Returns a collectable with hash.
## Examples
iex> collectable = Req.Utils.collect_with_hash([], :md5)
iex> Enum.into(Stream.duplicate("foo", 2), collectable)
{~w[foo foo], :erlang.md5("foofoo")}
"""
def collect_with_hash(collectable, type) do
%CollectWithHash{collectable: collectable, type: type}
end
@crlf "\r\n"
@doc """
Encodes fields into "multipart/form-data" format.
"""
def encode_form_multipart(fields, options \\ []) do
options = Keyword.validate!(options, [:boundary])
boundary =
options[:boundary] ||
Base.encode16(:crypto.strong_rand_bytes(16), padding: false, case: :lower)
footer = [[@crlf, "--", boundary, "--", @crlf]]
{body, size} =
fields
|> Enum.reduce({[], 0}, &add_form_parts(&2, encode_form_part(&1, boundary)))
|> add_form_parts({footer, IO.iodata_length(footer)})
%{
size: size,
content_type: "multipart/form-data; boundary=#{boundary}",
body: body
}
end
defp add_sizes(_, nil), do: nil
defp add_sizes(nil, _), do: nil
defp add_sizes(size1, size2), do: size1 + size2
defp add_form_parts({parts1, size1}, {parts2, size2})
when is_list(parts1) and is_list(parts2) do
{[parts1, parts2], add_sizes(size1, size2)}
end
defp add_form_parts({parts1, size1}, {parts2, size2}) do
{Stream.concat(parts1, parts2), add_sizes(size1, size2)}
end
defp encode_form_part({name, {value, options}}, boundary) do
options = Keyword.validate!(options, [:filename, :content_type, :size])
{parts, parts_size, options} =
case value do
integer when is_integer(integer) ->
part = Integer.to_string(integer)
{[part], byte_size(part), options}
value when is_binary(value) or is_list(value) ->
{[value], IO.iodata_length(value), options}
stream = %File.Stream{} ->
filename = Path.basename(stream.path)
# TODO: Simplify when we require Elixir v1.15
size =
if not Map.has_key?(stream, :node) or stream.node == node() do
File.stat!(stream.path).size
else
:erpc.call(stream.node, fn -> File.stat!(stream.path).size end)
end
options =
options
|> Keyword.put_new(:filename, filename)
|> Keyword.put_new_lazy(:content_type, fn ->
MIME.from_path(filename)
end)
{stream, size, options}
enum ->
size = Keyword.get(options, :size)
{enum, size, options}
end
params =
if filename = options[:filename] do
["; filename=\"", filename, "\""]
else
[]
end
headers =
if content_type = options[:content_type] do
["content-type: ", content_type, @crlf]
else
[]
end
headers = ["content-disposition: form-data; name=\"#{name}\"", params, @crlf, headers]
header = [[@crlf, "--", boundary, @crlf, headers, @crlf]]
add_form_parts({header, IO.iodata_length(header)}, {parts, parts_size})
end
defp encode_form_part({name, value}, boundary) do
encode_form_part({name, {value, []}}, boundary)
end
@doc """
Loads .netrc file.
## Examples
iex> {:ok, pid} = StringIO.open(\"""
...> machine localhost
...> login foo
...> password bar
...> \""")
iex> Req.Utils.load_netrc(pid)
%{"localhost" => {"foo", "bar"}}
"""
def load_netrc(path_or_device) do
case read_netrc(path_or_device) do
{:ok, ""} ->
raise ".netrc file is empty"
{:ok, contents} ->
contents
|> String.trim()
|> String.split()
|> parse_netrc()
{:error, reason} ->
raise "error reading .netrc file: #{:file.format_error(reason)}"
end
end
defp read_netrc(path) when is_binary(path) do
File.read(path)
end
defp read_netrc(pid) when is_pid(pid) do
<<content::binary>> = IO.read(pid, :eof)
{:ok, content}
end
defp parse_netrc(credentials), do: parse_netrc(credentials, %{})
defp parse_netrc([], acc), do: acc
defp parse_netrc([_, machine, _, login, _, password | tail], acc) do
acc = Map.put(acc, String.trim(machine), {String.trim(login), String.trim(password)})
parse_netrc(tail, acc)
end
defp parse_netrc(_, _), do: raise("error parsing .netrc file")
end