Packages

s3

0.1.0
0.1.1 0.1.0 0.1.0-rc.0

Minimal request builder for S3-compatible object storage API

HexDocs

Current section

3 Versions

Jump to
Readme
3 Versions
0 Dependencies
0 Dependants
Files
Activity
Hide whitespace

Compare versions

5 files changed
+82 additions
-34 deletions
  @@ -0,0 +1,5 @@
1 + # Changelog
2 +
3 + ## 0.1.0 (2024-03-31)
4 +
5 + - cleanup, upadte deps, add some docs
  @@ -1,9 +1,4 @@
1 - Small and experimental Amazon S3-compatible object storage "client" in a single file.
2 -
3 - Inspired by
4 - - https://gist.github.com/chrismccord/37862f1f8b1f5148644b75d20d1cb073 (single file, easy)
5 - - https://github.com/aws-beam/aws-elixir (clients are structs, xmerl)
6 - - https://github.com/ex-aws/ex_aws_s3 (streaming uploads and downloads)
1 + Small and experimental Amazon S3-compatible object storage request builder.
7 2
8 3 Verified to work with Amazon S3, MinIO.
9 4
  @@ -21,7 +16,7 @@ $ iex
21 16
22 17 ```elixir
23 18 # Setup
24 - Mix.install([:finch, {:s3, github: "ruslandoga/s3"}])
19 + Mix.install([:finch, :s3])
25 20 Finch.start_link(name: MinIO.Finch)
26 21
27 22 config = fn options ->
  @@ -142,7 +137,6 @@ req = Finch.build(:get, uri, headers, body)
142 137
143 138 # DeleteObjects
144 139 # https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
145 -
146 140 objects =
147 141 Enum.map(contents, fn {"Contents", contents} ->
148 142 {"Object", [List.keyfind!(contents, "Key", 0)]}
  @@ -153,7 +147,7 @@ content_md5 = Base.encode64(:crypto.hash(:md5, xml))
153 147
154 148 {uri, headers, body} =
155 149 S3.build(
156 - config(
150 + config.(
157 151 method: :post,
158 152 path: "/testbucket",
159 153 query: %{"delete" => ""},
  @@ -162,7 +156,6 @@ content_md5 = Base.encode64(:crypto.hash(:md5, xml))
162 156 )
163 157 )
164 158
165 -
166 159 {:ok,
167 160 {"DeleteResult",
168 161 [
  @@ -177,7 +170,7 @@ content_md5 = Base.encode64(:crypto.hash(:md5, xml))
177 170 # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-UsingHTTPPOST.html
178 171 # similar to https://gist.github.com/chrismccord/37862f1f8b1f5148644b75d20d1cb073
179 172
180 - # TODO what the api should be?
173 + # TODO what the api should be? S3.form/1
181 174 ```
182 175
183 176 ```elixir
  @@ -185,7 +178,7 @@ content_md5 = Base.encode64(:crypto.hash(:md5, xml))
185 178 # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
186 179
187 180 %URI{} = url =
188 - S3.signed_url(
181 + S3.url(
189 182 access_key_id: "AKIAIOSFODNN7EXAMPLE",
190 183 secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
191 184 region: "us-east-1",
  @@ -1,6 +1,6 @@
1 1 {<<"links">>,[{<<"GitHub">>,<<"https://github.com/ruslandoga/s3">>}]}.
2 2 {<<"name">>,<<"s3">>}.
3 - {<<"version">>,<<"0.1.0-rc.0">>}.
3 + {<<"version">>,<<"0.1.0">>}.
4 4 {<<"description">>,
5 5 <<"Minimal request builder for S3-compatible object storage API">>}.
6 6 {<<"elixir">>,<<"~> 1.15">>}.
  @@ -9,5 +9,5 @@
9 9 {<<"requirements">>,[]}.
10 10 {<<"files">>,
11 11 [<<"lib">>,<<"lib/s3.ex">>,<<".formatter.exs">>,<<"mix.exs">>,
12 - <<"README.md">>]}.
12 + <<"README.md">>,<<"CHANGELOG.md">>]}.
13 13 {<<"build_tools">>,[<<"mix">>]}.
  @@ -15,12 +15,12 @@ defmodule S3 do
15 15 | {:path, String.t()}
16 16 | {:query, Enumerable.t()}
17 17 | {:headers, headers}
18 - | {:body, iodata | {:stream, Enumerable.t()} | :url}
18 + # TODO | {:body, iodata | {:stream, Enumerable.t()}, :url} ?
19 + | {:body, iodata | {:stream, Enumerable.t()}}
19 20 | {:utc_now, DateTime.t()}
20 21
21 - @type options :: [option]
22 -
23 - @spec build(options) :: {URI.t(), headers, body :: iodata | Enumerable.t()}
22 + @doc "Builds URI, headers, body triplet to be used with HTTP clients."
23 + @spec build([option]) :: {URI.t(), headers, body :: iodata | Enumerable.t()}
24 24 def build(options) do
25 25 access_key_id = Keyword.fetch!(options, :access_key_id)
26 26 secret_access_key = Keyword.fetch!(options, :secret_access_key)
  @@ -55,7 +55,7 @@ defmodule S3 do
55 55
56 56 headers =
57 57 Enum.map(headers, fn {k, v} -> {String.downcase(k), v} end)
58 - |> put_header("host", host || url.host)
58 + |> put_header("host", host || url[:authority] || url.host)
59 59 |> put_header("x-amz-content-sha256", amz_content_sha256)
60 60 |> put_header("x-amz-date", amz_date)
61 61 |> Enum.sort_by(fn {k, _} -> k end)
  @@ -156,7 +156,8 @@ defmodule S3 do
156 156 {struct!(URI, url), headers, body}
157 157 end
158 158
159 - @spec signature(options) :: String.t()
159 + @doc "Calculates V4 signature"
160 + @spec signature([option]) :: String.t()
160 161 def signature(options) do
161 162 secret_access_key = Keyword.fetch!(options, :secret_access_key)
162 163 body = Keyword.fetch!(options, :body)
  @@ -175,8 +176,9 @@ defmodule S3 do
175 176 hex_hmac_sha256(signing_key, body)
176 177 end
177 178
178 - @spec signed_url(options) :: URI.t()
179 - def signed_url(options) do
179 + @doc "Returns a presigned URL"
180 + @spec sign([option]) :: URI.t()
181 + def sign(options) do
180 182 access_key_id = Keyword.fetch!(options, :access_key_id)
181 183 secret_access_key = Keyword.fetch!(options, :secret_access_key)
182 184
  @@ -207,7 +209,7 @@ defmodule S3 do
207 209
208 210 headers =
209 211 Enum.map(headers, fn {k, v} -> {String.downcase(k), v} end)
210 - |> put_header("host", host || url.host)
212 + |> put_header("host", host || url[:authority] || url.host)
211 213 |> Enum.sort_by(fn {k, _} -> k end)
212 214
213 215 path =
  @@ -331,12 +333,52 @@ defmodule S3 do
331 333
332 334 @type xml_element :: {String.t(), [xml_element() | String.t()]}
333 335
336 + @doc """
337 + Decodes XML binaries and encodes term to XML iodata.
338 +
339 + Examples:
340 +
341 + iex> xml("")
342 + ** (ArgumentError) Can't detect character encoding due to lack of indata
343 +
344 + iex> xml("<Hello></Hello>")
345 + {:ok, {"Hello", []}}
346 +
347 + iex> IO.iodata_to_binary(xml({"Hello", []}))
348 + "<Hello></Hello>"
349 +
350 + iex> xml(\"""
351 + ...> <book>
352 + ...>
353 + ...> <title> Learning Amazon Web Services </title>
354 + ...>
355 + ...> <author> Mark Wilkins </author>
356 + ...>
357 + ...> </book>
358 + ...> \""")
359 + {:ok, {"book", [{"title", [" Learning Amazon Web Services "]}, {"author", [" Mark Wilkins "]}]}}
360 +
361 + iex> IO.iodata_to_binary(xml({"book", [{"title", [" Learning Amazon Web Services "]}, {"author", [" Mark Wilkins "]}]}))
362 + "<book><title> Learning Amazon Web Services </title><author> Mark Wilkins </author></book>"
363 +
364 + iex> xml(\"""
365 + ...> <?xml version="1.0" encoding="UTF-8"?>
366 + ...> <俄语 լեզու="ռուսերեն">данные</俄语>
367 + ...> \""")
368 + {:ok, {"俄语", ["данные"]}}
369 +
370 + iex> IO.iodata_to_binary(xml({"俄语", ["данные"]}))
371 + "<俄语>данные</俄语>"
372 +
373 + """
334 374 @spec xml(binary) :: {:ok, xml_element} | {:error, any}
335 375 def xml(xml) when is_binary(xml) do
336 376 # TODO
337 377 # See: https://elixirforum.com/t/utf-8-issue-with-erlang-xmerl-scan-function/1668/9
338 378 # xml = :erlang.binary_to_list(xml)
339 379
380 + xml = String.trim(xml)
381 +
340 382 result =
341 383 :xmerl_sax_parser.stream(xml,
342 384 event_fun: &__MODULE__.xml_event_fun/3,
  @@ -345,7 +387,7 @@ defmodule S3 do
345 387
346 388 case result do
347 389 {:ok, xml, ""} -> {:ok, xml}
348 - # TODO incomplete or extra -> error
390 + {:fatal_error, _, reason, _, _} -> raise ArgumentError, List.to_string(reason)
349 391 {:error, _reason} = e -> e
350 392 end
351 393 end
  @@ -368,6 +410,14 @@ defmodule S3 do
368 410 [xml_escape(binary) | xml_continue(rest)]
369 411 end
370 412
413 + defp xml_continue([atom | rest]) when is_atom(atom) do
414 + [atom |> Atom.to_string() |> xml_escape() | xml_continue(rest)]
415 + end
416 +
417 + defp xml_continue([number | rest]) when is_number(number) do
418 + [to_string(number) | xml_continue(rest)]
419 + end
420 +
371 421 defp xml_continue([] = empty), do: empty
372 422
373 423 # TODO speed-up
  @@ -2,7 +2,7 @@ defmodule S3.MixProject do
2 2 use Mix.Project
3 3
4 4 @source_url "https://github.com/ruslandoga/s3"
5 - @version "0.1.0-rc.0"
5 + @version "0.1.0"
6 6
7 7 def project do
8 8 [
  @@ -28,13 +28,14 @@ defmodule S3.MixProject do
28 28 # Run "mix help deps" to learn about dependencies.
29 29 defp deps do
30 30 [
31 - {:finch, "~> 0.16.0", only: [:dev, :test]},
32 - {:jason, "~> 1.4", only: [:dev, :test]},
33 - {:aws_signature, "~> 0.3.1", only: [:test, :bench]},
31 + {:finch, "~> 0.18.0", only: [:dev, :test]},
32 + {:jason, "~> 1.4", only: [:dev, :test, :bench]},
33 + {:aws_signature, "~> 0.3.1", only: [:dev, :test, :bench]},
34 34 {:benchee, "~> 1.2", only: :bench},
35 - {:sweet_xml, "~> 0.7.4", only: :bench},
36 - {:saxy, "~> 1.5", only: :bench},
37 - {:meeseeks, "~> 0.17.0", only: :bench},
35 + {:sweet_xml, "~> 0.7.4", only: [:dev, :test, :bench]},
36 + {:saxy, "~> 1.5", only: [:dev, :test, :bench]},
37 + {:aws, "~> 0.14.1", only: [:dev, :test, :bench]},
38 + {:meeseeks, "~> 0.17.0", only: [:dev, :test, :bench]},
38 39 {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false},
39 40 {:ex_doc, ">= 0.0.0", only: :docs}
40 41 ]
  @@ -45,9 +46,8 @@ defmodule S3.MixProject do
45 46 source_url: @source_url,
46 47 source_ref: "v#{@version}",
47 48 main: "readme",
48 - extras: ["README.md"]
49 - # extras: ["README.md", "CHANGELOG.md"],
50 - # skip_undefined_reference_warnings_on: ["CHANGELOG.md"]
49 + extras: ["README.md", "CHANGELOG.md"],
50 + skip_undefined_reference_warnings_on: ["CHANGELOG.md"]
51 51 ]
52 52 end