Current section
Files
Jump to
Current section
Files
lib/arc/storage/gcs.ex
defmodule Arc.Storage.GCS do
alias GoogleApi.Storage.V1.{Api.Objects, Connection, Model.Object}
alias GoogleApi.Gax.{Request, Response}
@default_expiry_time 60 * 5
@endpoint "storage.googleapis.com"
@library_version Mix.Project.config() |> Keyword.get(:version, "")
# available options resource settings
# https://cloud.google.com/storage/docs/json_api/v1/objects#resource
@object_attrs [
"acl",
"cacheControl",
"contentDisposition",
"contentEncoding",
"contentLanguage",
"contentType",
"crc32c",
"etag",
"kmsKeyName",
"md5Hash",
"mediaLink",
"storageClass",
"temporaryHold",
"timeCreated",
"timeDeleted",
"timeStorageClassUpdated",
"updated"
]
def put(definition, version, {file, _scope} = file_and_scope) do
# Path must be calculated within put function as file.file_name has
# already been modified by arc/arc-ecto to reflect
# the definition's filename function
path =
definition
|> get_storage_dir(version, file_and_scope)
|> Path.join(file.file_name)
gcs_options =
get_gcs_options(definition, version, file_and_scope)
|> Enum.to_list()
|> Keyword.put(:acl, definition.acl(version, file_and_scope))
|> transform_headers()
|> to_object_attrs()
do_put(definition, file, path, gcs_options)
end
def url(definition, version, file_and_scope, options) do
key = gcs_key(definition, version, file_and_scope)
case Keyword.get(options, :signed, false) do
true -> build_signed_url(definition, key, options)
false -> build_url(definition, key)
end
end
defp build_signed_url(definition, endpoint, options) do
{:ok, client_id} = Goth.Config.get("client_email")
expiration = System.os_time(:second) + Keyword.get(options, :expires_in, @default_expiry_time)
path = build_path(definition, endpoint)
signature_string = url_to_sign("GET", "", "", expiration, "", path)
url_encoded_signature = base64_sign_url(signature_string)
base_url = build_url(definition, endpoint)
"#{base_url}?GoogleAccessId=#{client_id}&Expires=#{expiration}&Signature=#{url_encoded_signature}"
end
def delete(definition, version, file_and_scope) do
key = gcs_key(definition, version, file_and_scope)
bucket = bucket_name(definition)
Objects.storage_objects_delete(conn(), bucket, key)
end
defp get_storage_dir(definition, version, file_and_scope) do
version
|> definition.storage_dir(file_and_scope)
|> gcs_storage_dir()
end
defp gcs_storage_dir({:system, env_value}) when is_binary(env_value) do
System.get_env(env_value)
end
defp gcs_storage_dir(name) do
name
end
defp do_put(definition, %Arc.File{binary: nil} = file, path, gcs_options) do
obj = build_object({file, path}, gcs_options)
bucket = bucket_name(definition)
insert_opts =
case obj.acl do
false -> []
acl -> [predefinedAcl: acl]
end
Objects.storage_objects_insert_simple(
conn(),
bucket,
"multipart",
obj,
file.path,
insert_opts
)
end
defp do_put(definition, %Arc.File{binary: binary} = file, path, gcs_options) do
obj = build_object({file, path}, gcs_options)
bucket = bucket_name(definition)
body =
Tesla.Multipart.new()
|> Tesla.Multipart.add_field(
"metadata",
Poison.encode!(obj),
headers: [{:"Content-Type", "application/json"}]
)
|> Tesla.Multipart.add_file_content(binary, "data")
request =
Request.new()
|> Request.method(:post)
|> Request.url("/upload/storage/v1/b/{bucket}/o", %{
"bucket" => URI.encode(bucket, &URI.char_unreserved?/1)
})
|> Request.add_param(:query, :uploadType, "multipart")
|> Request.add_param(:body, :body, body)
|> add_acl_param(obj.acl)
|> Request.library_version(@library_version)
conn()
|> Connection.execute(request)
|> Response.decode(struct: %Object{})
end
defp add_acl_param(request, false), do: request
defp add_acl_param(request, acl) do
Request.add_optional_params(request, %{predefinedAcl: :query}, predefinedAcl: acl)
end
defp transform_headers(headers) do
Enum.map(headers, &transform_header/1)
end
defp transform_header({key, val}) when key in [:acl, "acl"] and val != false do
{camelize(key), camelize(val)}
end
defp transform_header({key, val}) do
{camelize(key), val}
end
defp to_object_attrs(headers) do
Enum.reduce(headers, [], &to_object_attrs/2)
end
defp to_object_attrs({key, val}, acc) when key in @object_attrs do
[{String.to_atom(key), val} | acc]
end
defp to_object_attrs(_, acc) do
acc
end
defp endpoint do
case Application.fetch_env(:arc, :asset_host) do
:error -> @endpoint
{:ok, {:system, env_var}} when is_binary(env_var) -> System.get_env(env_var)
{:ok, endpoint} -> endpoint
end
end
defp gcs_key(definition, version, file_and_scope) do
definition
|> do_gcs_key(version, file_and_scope)
end
defp do_gcs_key(definition, version, file_and_scope) do
Path.join([
get_storage_dir(definition, version, file_and_scope),
Arc.Definition.Versioning.resolve_file_name(definition, version, file_and_scope)
])
end
defp get_gcs_options(definition, version, {file, scope}) do
try do
apply(definition, :gcs_object_headers, [version, {file, scope}])
rescue
UndefinedFunctionError ->
[]
end
end
defp build_url(definition, path) do
%URI{
host: endpoint(),
path: build_path(definition, path),
scheme: "https"
}
|> URI.to_string()
end
defp build_path(definition, path) do
case bucket_name(definition) do
nil -> path
value -> Path.join(value, path)
end
|> prepend_slash()
|> URI.encode()
end
defp bucket_name(definition) do
with {:system, env_var} <- definition.bucket() do
System.get_env(to_string(env_var))
end
end
defp prepend_slash("/" <> _rest = path), do: path
defp prepend_slash(path), do: "/#{path}"
defp url_to_sign(verb, md5, type, expiration, headers, resource) do
"#{verb}\n#{md5}\n#{type}\n#{expiration}\n#{headers}#{resource}"
end
defp base64_sign_url(plaintext) do
{:ok, pem_bin} = Goth.Config.get("private_key")
[pem_key_data] = :public_key.pem_decode(pem_bin)
otp_release = System.otp_release() |> String.to_integer()
rsa_key =
case otp_release do
n when n >= 21 ->
:public_key.pem_entry_decode(pem_key_data)
n when n <= 20 ->
pem_key = :public_key.pem_entry_decode(pem_key_data)
:public_key.der_decode(:RSAPrivateKey, elem(pem_key, 3))
end
plaintext
|> :public_key.sign(:sha256, rsa_key)
|> Base.encode64()
|> URI.encode_www_form()
end
defp build_object({%Arc.File{file_name: file_name}, name}, opts) do
opts
|> Keyword.put(:name, name)
|> Keyword.put_new(:contentType, MIME.from_path(file_name))
|> (&struct(Object, &1)).()
end
defp conn(), do: Connection.new(&for_scope/1)
defp camelize(word) do
case Regex.split(~r/(?:^|[-_])|(?=[A-Z])/, to_string(word), trim: true) do
[h] -> [String.downcase(h)]
[h | t] -> [String.downcase(h), Enum.map(t, &String.capitalize/1)]
[] -> []
end
|> Enum.join()
end
defp for_scope(scopes) do
token_store = Application.get_env(:arc, :token_fetcher, Arc.Storage.GCS.Token.DefaultFetcher)
token_store.get_token(scopes)
end
end