Packages

phoenix_kit

1.7.14
1.7.207 1.7.206 1.7.205 1.7.204 1.7.203 1.7.202 1.7.201 1.7.200 1.7.199 1.7.198 1.7.197 1.7.196 1.7.194 1.7.193 1.7.192 1.7.191 1.7.190 1.7.189 1.7.187 1.7.186 1.7.185 1.7.184 1.7.183 1.7.182 1.7.181 1.7.180 1.7.179 1.7.178 1.7.177 1.7.176 1.7.175 1.7.174 1.7.173 1.7.172 1.7.171 1.7.170 1.7.169 1.7.168 1.7.167 1.7.166 1.7.165 1.7.164 1.7.162 1.7.161 1.7.160 1.7.159 1.7.157 1.7.156 1.7.155 1.7.154 1.7.153 1.7.152 1.7.151 1.7.150 1.7.149 1.7.146 1.7.145 1.7.144 1.7.143 1.7.138 1.7.133 1.7.132 1.7.131 1.7.130 1.7.128 1.7.126 1.7.125 1.7.121 1.7.120 1.7.119 1.7.118 1.7.117 1.7.116 1.7.115 1.7.114 1.7.113 1.7.112 1.7.111 1.7.110 1.7.109 1.7.108 1.7.107 1.7.106 1.7.105 1.7.104 1.7.103 1.7.102 1.7.101 1.7.100 1.7.99 1.7.98 1.7.97 1.7.96 1.7.95 1.7.94 1.7.93 1.7.92 1.7.91 1.7.90 1.7.89 1.7.88 1.7.87 1.7.86 1.7.85 1.7.84 1.7.83 1.7.82 1.7.81 1.7.80 1.7.79 1.7.78 1.7.77 1.7.76 1.7.75 1.7.74 1.7.71 1.7.70 1.7.69 1.7.66 1.7.65 1.7.64 1.7.63 1.7.62 1.7.61 1.7.59 1.7.58 1.7.57 1.7.56 1.7.55 1.7.54 1.7.53 1.7.52 1.7.51 1.7.49 1.7.44 1.7.43 1.7.42 1.7.41 1.7.39 1.7.38 1.7.37 1.7.36 1.7.34 1.7.33 1.7.31 1.7.30 1.7.29 1.7.28 1.7.27 1.7.26 1.7.25 1.7.24 1.7.23 1.7.22 1.7.21 1.7.20 1.7.19 1.7.18 1.7.17 1.7.16 1.7.15 1.7.14 1.7.13 1.7.12 1.7.11 1.7.10 1.7.9 1.7.8 1.7.7 1.7.6 1.7.5 1.7.4 1.7.3 1.7.2 1.7.1 1.7.0 1.6.20 1.6.19 1.6.18 1.6.17 1.6.16 1.6.15 1.6.14 1.6.13 1.6.12 1.6.11 1.6.10 1.6.9 1.6.8 1.6.7 1.6.6 1.6.5 1.6.4 1.6.3 1.5.2 1.5.1 1.5.0 1.4.9 1.4.8 1.4.7 1.4.6 1.4.5 1.4.4 1.4.3 1.4.2 1.4.1 1.4.0 1.3.2 1.3.1 1.3.0 1.2.10 1.2.9 1.2.8 1.2.7 1.2.5 1.2.4 1.2.2 1.2.1 1.2.0 1.1.0 1.0.0

A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more

Current section

Files

Jump to
phoenix_kit lib modules storage services manager.ex
Raw

lib/modules/storage/services/manager.ex

defmodule PhoenixKit.Modules.Storage.Manager do
@moduledoc """
Storage manager for handling file operations with redundancy and failover.
This module coordinates file storage across multiple buckets with automatic
redundancy, failover, and variant generation.
"""
alias PhoenixKit.Modules.Storage
alias PhoenixKit.Modules.Storage.ProviderRegistry
alias PhoenixKit.Settings
# Cache TTL for bucket list (5 minutes)
@buckets_cache_ttl 300_000
@doc """
Stores a file across multiple buckets based on redundancy settings.
## Options
- `:redundancy_copies` - Number of copies to store (default: from settings)
- `:priority_buckets` - List of specific bucket IDs to use (default: auto-select)
- `:force_bucket_ids` - List of specific bucket IDs to use (overrides priority_buckets)
- `:generate_variants` - Whether to generate variants (default: from settings)
## Returns
- `{:ok, file_result}` - File stored successfully with locations
- `{:error, reason}` - Failed to store file
"""
def store_file(source_path, opts \\ []) do
# Get redundancy settings
redundancy_copies = Keyword.get(opts, :redundancy_copies, get_redundancy_copies())
force_bucket_ids = Keyword.get(opts, :force_bucket_ids, [])
priority_buckets = Keyword.get(opts, :priority_buckets, [])
_generate_variants = Keyword.get(opts, :generate_variants, get_auto_generate_variants())
# Use force_bucket_ids if provided, otherwise use priority_buckets
buckets_to_use =
if Enum.empty?(force_bucket_ids), do: priority_buckets, else: force_bucket_ids
# Select buckets for storage
buckets = select_buckets_for_storage(redundancy_copies, buckets_to_use)
if Enum.empty?(buckets) do
{:error, "No available storage buckets"}
else
# Store file across selected buckets
store_across_buckets(source_path, buckets, opts)
end
rescue
error -> {:error, "Error storing file: #{inspect(error)}"}
end
@doc """
Retrieves a file from storage with failover.
Tries each bucket in priority order until the file is found.
"""
def retrieve_file(file_path, opts \\ []) do
priority_buckets = Keyword.get(opts, :priority_buckets, [])
buckets = select_buckets_for_retrieval(priority_buckets)
retrieve_with_failover(file_path, buckets, opts)
end
@doc """
Deletes a file from all storage buckets.
"""
def delete_file(file_path, opts \\ []) do
priority_buckets = Keyword.get(opts, :priority_buckets, [])
buckets = select_buckets_for_retrieval(priority_buckets)
results =
buckets
|> Enum.map(fn bucket ->
provider = get_provider_for_bucket(bucket)
provider.delete_file(bucket, file_path)
end)
# Return success if at least one deletion succeeded
if Enum.any?(results, &(&1 == :ok)) do
:ok
else
{:error, "Failed to delete file from all buckets"}
end
end
@doc """
Checks if a file exists in any storage bucket.
"""
def file_exists?(file_path, opts \\ []) do
priority_buckets = Keyword.get(opts, :priority_buckets, [])
buckets = select_buckets_for_retrieval(priority_buckets)
Enum.any?(buckets, fn bucket ->
provider = get_provider_for_bucket(bucket)
provider.file_exists?(bucket, file_path)
end)
end
@doc """
Gets a public URL for a file from the highest priority bucket that has it.
"""
def public_url(file_path, opts \\ []) do
priority_buckets = Keyword.get(opts, :priority_buckets, [])
buckets = select_buckets_for_retrieval(priority_buckets)
Enum.find_value(buckets, fn bucket ->
provider = get_provider_for_bucket(bucket)
if provider.file_exists?(bucket, file_path) do
provider.public_url(bucket, file_path)
else
nil
end
end)
end
# Private functions
defp select_buckets_for_storage(redundancy_copies, priority_buckets) do
if Enum.empty?(priority_buckets) do
# Get fresh bucket list from database (don't use cache for selection)
# This ensures we get the current state and can shuffle properly
all_buckets = Storage.list_enabled_buckets()
# Separate buckets by priority
{auto_priority_buckets, fixed_priority_buckets} =
Enum.split_with(all_buckets, &(&1.priority == 0))
# Shuffle auto-priority buckets (priority = 0) for random distribution
# Fixed priority buckets are deterministic
shuffled_auto = Enum.shuffle(auto_priority_buckets)
# Combine: fixed priority buckets first (sorted), then shuffled auto-priority
(Enum.sort_by(fixed_priority_buckets, & &1.priority) ++ shuffled_auto)
|> Enum.take(redundancy_copies)
else
# Use specified buckets
Storage.list_enabled_buckets()
|> Enum.filter(&(&1.id in priority_buckets))
|> Enum.take(redundancy_copies)
end
end
defp select_buckets_for_retrieval(priority_buckets) do
if Enum.empty?(priority_buckets) do
# Use all enabled buckets ordered by priority (simple sort, no usage calculation needed for retrieval)
get_enabled_buckets()
|> Enum.sort_by(& &1.priority)
else
# Use specified buckets
get_enabled_buckets()
|> Enum.filter(&(&1.id in priority_buckets))
end
end
defp store_across_buckets(source_path, buckets, opts) do
# Use path_prefix if provided, otherwise generate a path
destination_path =
case Keyword.get(opts, :path_prefix) do
nil -> generate_destination_path(source_path, opts)
path_prefix -> path_prefix
end
results =
buckets
|> Enum.map(fn bucket ->
provider = get_provider_for_bucket(bucket)
provider.store_file(bucket, source_path, destination_path, opts)
end)
# Check if at least one storage succeeded
successful_storages = Enum.count(results, &(&1 == :ok or match?({:ok, _}, &1)))
if successful_storages > 0 do
file_info = %{
destination_path: destination_path,
stored_in: length(buckets),
successful_storages: successful_storages,
bucket_ids: Enum.map(buckets, & &1.id)
}
{:ok, file_info}
else
{:error, "Failed to store file in any bucket"}
end
end
defp retrieve_with_failover(_file_path, [], _opts), do: {:error, "File not found in any bucket"}
defp retrieve_with_failover(file_path, [bucket | remaining_buckets], opts) do
provider = get_provider_for_bucket(bucket)
destination_path = Keyword.get(opts, :destination_path, generate_temp_path())
case provider.retrieve_file(bucket, file_path, destination_path) do
:ok ->
{:ok, destination_path}
{:error, _reason} ->
retrieve_with_failover(file_path, remaining_buckets, opts)
end
end
defp get_provider_for_bucket(bucket) do
{:ok, provider_module} = ProviderRegistry.get_provider(bucket.provider)
provider_module
end
defp generate_destination_path(source_path, opts) do
original_name = Path.basename(source_path)
extension = Path.extname(original_name)
base_name = Path.rootname(original_name)
timestamp = DateTime.utc_now() |> DateTime.to_iso8601()
random_suffix = :crypto.strong_rand_bytes(4) |> Base.encode16(case: :lower)
prefix = Keyword.get(opts, :path_prefix, "")
subdir = Keyword.get(opts, :subdir, timestamp)
Path.join([prefix, subdir, "#{base_name}_#{random_suffix}#{extension}"])
end
defp generate_temp_path do
temp_dir = System.tmp_dir!()
random_name = :crypto.strong_rand_bytes(8) |> Base.encode16(case: :lower)
Path.join(temp_dir, "phoenix_kit_#{random_name}")
end
defp get_enabled_buckets do
# Cache bucket list to avoid querying on every file request
cache_key = :phoenix_kit_buckets_cache
current_time = System.monotonic_time(:millisecond)
case :persistent_term.get(cache_key, nil) do
{timestamp, buckets} when current_time - timestamp < @buckets_cache_ttl ->
# Cache hit - return cached buckets
buckets
_ ->
# Cache miss or expired - fetch fresh buckets
buckets = Storage.list_enabled_buckets()
:persistent_term.put(cache_key, {current_time, buckets})
buckets
end
end
defp get_redundancy_copies do
Settings.get_setting_cached("storage_redundancy_copies", "1")
|> String.to_integer()
|> max(1)
|> min(5)
end
defp get_auto_generate_variants do
Settings.get_setting_cached("storage_auto_generate_variants", "true") == "true"
end
end