Packages
phoenix_kit
2.59.0
2.60.1
2.60.0
2.59.0
2.58.0
2.57.1
2.57.0
2.56.1
2.56.0
2.55.1
2.55.0
2.54.2
2.54.1
2.54.0
2.53.0
2.52.2
2.52.1
2.52.0
2.51.0
2.50.0
2.49.1
2.49.0
2.48.0
2.47.0
2.46.0
2.45.0
2.44.0
2.43.1
2.43.0
2.42.1
2.42.0
2.41.6
2.41.4
2.41.3
2.41.2
2.41.1
2.41.0
2.40.1
2.40.0
2.39.0
2.38.1
2.38.0
2.37.5
2.37.4
2.37.3
2.37.2
2.37.1
2.37.0
2.36.1
2.36.0
2.35.0
2.34.0
2.33.0
2.32.1
2.32.0
2.31.1
2.31.0
2.30.0
2.29.1
2.29.0
2.28.2
2.28.1
2.28.0
2.27.2
2.27.1
2.27.0
2.26.1
2.26.0
2.25.0
2.24.0
2.23.3
2.23.2
2.23.1
2.23.0
2.22.24
2.22.23
2.22.22
2.22.21
2.22.20
2.22.19
2.22.18
2.22.17
2.22.16
2.22.15
2.22.14
2.22.13
2.22.12
2.22.11
2.22.10
2.22.9
2.22.8
2.22.7
2.22.6
2.22.5
2.22.4
2.22.3
2.22.2
2.22.1
2.22.0
2.21.5
2.21.4
2.21.3
2.21.2
2.21.1
2.21.0
2.20.0
2.19.0
2.18.1
2.18.0
2.17.0
2.16.0
2.15.1
2.15.0
2.14.2
2.14.1
2.14.0
2.13.19
2.13.18
2.13.17
2.13.16
2.13.15
2.13.13
2.13.12
2.13.11
2.13.10
2.13.9
2.13.8
2.13.7
2.13.6
2.13.5
2.13.4
2.13.3
2.13.2
2.13.1
2.13.0
2.12.1
2.12.0
2.11.0
2.10.0
2.9.0
2.8.1
2.8.0
2.7.0
2.6.0
2.5.0
2.4.0
2.3.0
2.2.0
2.1.0
2.0.1
2.0.0
1.7.236
1.7.235
1.7.234
1.7.233
1.7.232
1.7.231
1.7.230
1.7.229
1.7.228
1.7.227
1.7.226
1.7.225
1.7.224
1.7.223
1.7.222
1.7.221
1.7.220
1.7.219
1.7.218
1.7.217
1.7.216
1.7.215
1.7.214
1.7.213
1.7.212
1.7.211
1.7.210
1.7.209
1.7.208
1.7.207
1.7.206
1.7.205
1.7.204
1.7.203
1.7.202
1.7.201
1.7.200
1.7.199
1.7.198
1.7.197
1.7.196
1.7.194
1.7.193
1.7.192
1.7.191
1.7.190
1.7.189
1.7.187
1.7.186
1.7.185
1.7.184
1.7.183
1.7.182
1.7.181
1.7.180
1.7.179
1.7.178
1.7.177
1.7.176
1.7.175
1.7.174
1.7.173
1.7.172
1.7.171
1.7.170
1.7.169
1.7.168
1.7.167
1.7.166
1.7.165
1.7.164
1.7.162
1.7.161
1.7.160
1.7.159
1.7.157
1.7.156
1.7.155
1.7.154
1.7.153
1.7.152
1.7.151
1.7.150
1.7.149
1.7.146
1.7.145
1.7.144
1.7.143
1.7.138
1.7.133
1.7.132
1.7.131
1.7.130
1.7.128
1.7.126
1.7.125
1.7.121
1.7.120
1.7.119
1.7.118
1.7.117
1.7.116
1.7.115
1.7.114
1.7.113
1.7.112
1.7.111
1.7.110
1.7.109
1.7.108
1.7.107
1.7.106
1.7.105
1.7.104
1.7.103
1.7.102
1.7.101
1.7.100
1.7.99
1.7.98
1.7.97
1.7.96
1.7.95
1.7.94
1.7.93
1.7.92
1.7.91
1.7.90
1.7.89
1.7.88
1.7.87
1.7.86
1.7.85
1.7.84
1.7.83
1.7.82
1.7.81
1.7.80
1.7.79
1.7.78
1.7.77
1.7.76
1.7.75
1.7.74
1.7.71
1.7.70
1.7.69
1.7.66
1.7.65
1.7.64
1.7.63
1.7.62
1.7.61
1.7.59
1.7.58
1.7.57
1.7.56
1.7.55
1.7.54
1.7.53
1.7.52
1.7.51
1.7.49
1.7.44
1.7.43
1.7.42
1.7.41
1.7.39
1.7.38
1.7.37
1.7.36
1.7.34
1.7.33
1.7.31
1.7.30
1.7.29
1.7.28
1.7.27
1.7.26
1.7.25
1.7.24
1.7.23
1.7.22
1.7.21
1.7.20
1.7.19
1.7.18
1.7.17
1.7.16
1.7.15
1.7.14
1.7.13
1.7.12
1.7.11
1.7.10
1.7.9
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.20
1.6.19
1.6.18
1.6.17
1.6.16
1.6.15
1.6.14
1.6.13
1.6.12
1.6.11
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.5.2
1.5.1
1.5.0
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.2
1.3.1
1.3.0
1.2.10
1.2.9
1.2.8
1.2.7
1.2.5
1.2.4
1.2.2
1.2.1
1.2.0
1.1.0
1.0.0
A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more
Current section
Files
Jump to
Current section
Files
lib/modules/storage/focal_point.ex
defmodule PhoenixKit.Modules.Storage.FocalPoint do
@moduledoc """
Where the subject of a photo is, so a rendition can crop around it.
A **focal point** is `{x, y}`, fractions of the photo **as it is displayed**
(after its EXIF orientation), `0.0..1.0` from the left and from the top. It is
kept in the file's `metadata` JSON (`"focal"`: `x`, `y` and `source`), so it
needs no column. A fixed rendition whose crop mode is `focus` (V210) crops a
window of its shape centered on it (`ImageProcessor.resize_and_crop_focus/6`);
every such rendition of the photo (a square thumbnail, a 4:3, a 16:9) uses the
same point, and a photo with none is cropped at the center.
## Where it comes from
* `"manual"`: set by a person (`put/4`); wins over everything and is never
replaced by detection.
* `"auto"`: found by `detect/1`, which asks libvips where the photo draws the
eye (`smartcrop` with `attention`: edges, colour, skin tone) on a copy shrunk to
#{512}px, which costs a few milliseconds. It needs the optional `vix`
package; without it, or when detection fails, there is no point and the crop
stays at the center.
`ensure/2` is what the generator calls the first time a photo needs one: the
stored point, else a detected one (stored for next time), else `nil`.
"""
import Ecto.Query, only: [from: 2]
alias PhoenixKit.Modules.Storage
alias PhoenixKit.Modules.Storage.Dimension
alias PhoenixKit.Modules.Storage.File, as: StorageFile
alias PhoenixKit.Modules.Storage.FileInstance
alias PhoenixKit.Modules.Storage.ImageProcessor
alias PhoenixKit.Modules.Storage.VariantGenerator
alias PhoenixKit.Modules.Storage.VariantSets
alias PhoenixKit.RepoHelper
require Logger
# Hosts without the optional dependency must compile without warnings too.
@compile {:no_warn_undefined, Vix.Vips.Image}
@compile {:no_warn_undefined, Vix.Vips.Operation}
# The long side of the copy detection looks at.
@shrink_to 512
# The same ceiling the ImageMagick calls keep: no decoding a decompression bomb.
@max_pixels 40_000_000
@sources ~w(auto manual)
@type point :: {float(), float()}
@doc "Whether detection is available (the optional `vix` package is loaded)."
@spec detection_available?() :: boolean()
def detection_available?, do: Code.ensure_loaded?(Vix.Vips.Operation)
@doc """
The focal point stored on `file` and where it came from, or `nil`:
`{{x, y}, "auto" | "manual"}`.
"""
@spec get(map()) :: {point(), String.t()} | nil
def get(%{metadata: metadata}), do: from_metadata(metadata)
def get(_file), do: nil
defp from_metadata(%{"focal" => %{"x" => x, "y" => y} = focal})
when is_number(x) and is_number(y) and x >= 0 and x <= 1 and y >= 0 and y <= 1 do
{{x / 1, y / 1}, Map.get(focal, "source", "auto")}
end
defp from_metadata(_metadata), do: nil
@doc """
Records a focal point for `file`: `x` and `y` between 0 and 1, `source` `"auto"`
or `"manual"`. A detected point never replaces a manual one (`{:ok, :kept}`).
Only the `"focal"` key of the metadata changes. A change of point also
invalidates generated subject crops and queues reconciliation.
Detected points are accepted only while the file's checksum is unchanged.
Pass `source_key:` when the bytes came from `Storage.retrieve_original/1`;
that original must still belong to the file when the point is recorded.
"""
@spec put(map(), number(), number(), String.t(), keyword()) ::
{:ok, point()} | {:ok, :kept} | {:error, :invalid | :stale_source}
def put(file, x, y, source, opts \\ [])
def put(%{uuid: uuid} = file, x, y, source, opts)
when is_number(x) and is_number(y) and x >= 0 and x <= 1 and y >= 0 and y <= 1 and
source in @sources do
focal = %{"x" => Float.round(x / 1, 4), "y" => Float.round(y / 1, 4), "source" => source}
# `source = auto` writes only where no manual point is: one statement, so a
# person's choice made while detection runs is not lost.
query =
if source == "manual" do
from(f in StorageFile, where: f.uuid == ^uuid)
else
from(f in StorageFile,
where:
f.uuid == ^uuid and
fragment("coalesce(?->'focal'->>'source', '') <> 'manual'", f.metadata)
)
end
RepoHelper.repo().transaction(fn ->
current =
RepoHelper.repo().one(from(f in StorageFile, where: f.uuid == ^uuid, lock: "FOR UPDATE"))
check_source!(file, current, source, Keyword.get(opts, :source_key))
{count, _} =
from(f in query,
update: [
set: [
metadata:
fragment(
"coalesce(?, '{}'::jsonb) || ?",
f.metadata,
type(^%{"focal" => focal}, :map)
)
]
]
)
|> RepoHelper.repo().update_all([])
point = {focal["x"], focal["y"]}
if count == 1 do
if point_from_file(current) != point, do: invalidate_crops(current)
point
else
:kept
end
end)
end
def put(_file, _x, _y, _source, _opts), do: {:error, :invalid}
defp check_source!(file, current, "auto", source_key) when not is_nil(current) do
checksum_changed? =
Map.has_key?(file, :file_checksum) and current.file_checksum != file.file_checksum
original_changed? =
not is_nil(source_key) and not Storage.original_key?(file.uuid, source_key)
if checksum_changed? or original_changed?, do: RepoHelper.repo().rollback(:stale_source)
end
defp check_source!(_file, _current, _source, _source_key), do: :ok
@doc "Forgets the stored focal point and requests regeneration of its subject crops."
@spec clear(map()) :: :ok
def clear(%{uuid: uuid}) do
RepoHelper.repo().transaction(fn ->
current =
RepoHelper.repo().one(from(f in StorageFile, where: f.uuid == ^uuid, lock: "FOR UPDATE"))
from(f in StorageFile,
where: f.uuid == ^uuid,
update: [set: [metadata: fragment("coalesce(?, '{}'::jsonb) - 'focal'", f.metadata)]]
)
|> RepoHelper.repo().update_all([])
if point_from_file(current), do: invalidate_crops(current)
end)
:ok
end
defp point_from_file(file) do
case get(file) do
{point, _source} -> point
nil -> nil
end
end
# Changing the point changes the pixels without changing a rendition's spec.
# Invalidate only generated focus crops (never an annotation with no spec).
defp invalidate_crops(file) do
names =
file
|> VariantGenerator.expected_variants()
|> Enum.filter(fn {dimension, _name, _format} -> Dimension.focus_crop?(dimension) end)
|> Enum.map(fn {_dimension, name, _format} -> name end)
{count, _} =
from(i in FileInstance,
where: i.file_uuid == ^file.uuid and i.variant_name in ^names and not is_nil(i.spec_hash)
)
|> RepoHelper.repo().update_all(set: [spec_hash: "focal_changed"])
if count > 0, do: VariantSets.record_variants(file, false, nil)
end
@doc """
The focal point to crop `file` around, from its original at `local_path`: the
stored one, else a detected one (stored, so the next rendition does not detect
again), else `nil`. Never raises: a photo detection cannot read is cropped at
the center.
"""
@spec ensure(map(), Path.t(), keyword()) :: point() | nil
def ensure(%{uuid: uuid} = file, local_path, opts \\ []) do
# Read again: a person may have set one since `file` was loaded.
stored =
RepoHelper.repo().one(from(f in StorageFile, where: f.uuid == ^uuid, select: f.metadata))
case from_metadata(stored) do
{point, _source} ->
point
nil ->
source_key =
Keyword.get_lazy(opts, :source_key, fn ->
case Storage.get_file_instance_by_name(uuid, "original") do
%{file_name: key} -> key
nil -> nil
end
end)
detect_and_store(file, local_path, source_key)
end
rescue
error ->
Logger.warning("FocalPoint.ensure failed: #{Exception.message(error)}")
nil
catch
:exit, reason ->
Logger.warning("FocalPoint.ensure failed: #{inspect(reason)}")
nil
end
defp detect_and_store(file, local_path, source_key) do
case detect(local_path) do
{:ok, {x, y}} ->
case put(file, x, y, "auto", source_key: source_key) do
{:ok, {_x, _y} = point} -> point
# A manual point appeared meanwhile.
{:ok, :kept} -> get_stored(file)
{:error, :stale_source} -> nil
_ -> nil
end
:error ->
nil
end
end
defp get_stored(%{uuid: uuid}) do
stored =
RepoHelper.repo().one(from(f in StorageFile, where: f.uuid == ^uuid, select: f.metadata))
case from_metadata(stored) do
{point, _source} -> point
nil -> nil
end
end
@doc """
Where the photo at `path` draws the eye, as `{:ok, {x, y}}` (fractions of the
displayed photo), or `:error` when detection is unavailable or cannot read it.
"""
@spec detect(Path.t()) :: {:ok, point()} | :error
def detect(path) do
if detection_available?(), do: run_detection(path), else: :error
end
defp run_detection(path) do
with {:ok, {w, h}} <- ImageProcessor.extract_dimensions(path),
true <- w * h <= @max_pixels do
# A NIF call: bounded in time, and a failure is "no focal point".
#
# The preview path is made here, not in the task: a killed task skips its
# own cleanup, and the file must not outlive a timeout.
preview = preview_path()
try do
task = Task.async(fn -> safe_attention(path, preview) end)
case Task.yield(task, 10_000) || Task.shutdown(task, :brutal_kill) do
{:ok, result} ->
result
_timeout_or_exit ->
# Killing the task does not stop the converter it was waiting for:
# that process runs on and may write the preview after the `after`
# below has run. It is bounded by ImageMagick's own time limit
# (`ImageProcessor.limit_args/0`), so look once more after that.
remove_preview_later(preview)
:error
end
after
File.rm(preview)
end
else
_ -> :error
end
rescue
_ -> :error
catch
_, _ -> :error
end
# A linked task must catch decoder failures itself: rescuing in the caller
# does not prevent a task's exception from exiting the caller too.
#
# A photo libvips cannot decode (an iPhone's HEIC: the precompiled library has
# no HEVC decoder, which ImageMagick has) is not "no subject": it is looked at
# again through a small JPEG preview ImageMagick makes of it.
defp safe_attention(path, preview) do
case try_attention(path) do
:unreadable -> from_preview(path, preview)
found -> found
end
end
defp try_attention(path) do
attention(path)
rescue
_ -> :unreadable
catch
_, _ -> :unreadable
end
# ImageMagick's `-limit time` (60 s) plus a margin.
@converter_limit_ms 70_000
defp remove_preview_later(preview) do
Task.start(fn ->
Process.sleep(@converter_limit_ms)
File.rm(preview)
end)
end
defp preview_path do
Path.join(
System.tmp_dir!(),
"phoenix_kit_focal_#{Base.encode16(:crypto.strong_rand_bytes(6), case: :lower)}.jpg"
)
end
# The caller removes `preview` once the task is done or killed.
defp from_preview(path, preview) do
with {:ok, _} <- ImageProcessor.preview_jpeg(path, preview, @shrink_to),
found when found != :unreadable <- try_attention(preview) do
Logger.info("FocalPoint: libvips could not decode a photo; used an ImageMagick preview")
found
else
_ -> :error
end
end
# Shrink on load (which also applies the EXIF orientation), then ask where the
# attention is. The coordinates are on the shrunk copy, so they are divided by
# its size.
defp attention(path) do
alias Vix.Vips.{Image, Operation}
with {:ok, small} <- Operation.thumbnail(path, @shrink_to, size: :VIPS_SIZE_DOWN),
width = Image.width(small),
height = Image.height(small),
true <- width > 1 and height > 1,
{:ok, {_cropped, found}} <-
Operation.smartcrop(small, max(div(width, 2), 1), max(div(height, 2), 1),
interesting: :VIPS_INTERESTING_ATTENTION
) do
found = Map.new(found)
x = found[:"attention-x"]
y = found[:"attention-y"]
cond do
not (is_number(x) and is_number(y)) -> :error
# A photo with nothing that draws the eye (flat, or blank) is reported at
# the corner: that is "nothing found", not a subject in the corner.
x == 0 and y == 0 -> :error
true -> {:ok, {clamp(x / width), clamp(y / height)}}
end
else
# Too small to look at: nothing to find, and nothing a preview would change.
false -> :error
# libvips could not open or decode it.
_ -> :unreadable
end
end
defp clamp(value), do: value |> max(0.0) |> min(1.0) |> Kernel./(1)
end