Packages
phoenix_kit
1.7.76
1.7.207
1.7.206
1.7.205
1.7.204
1.7.203
1.7.202
1.7.201
1.7.200
1.7.199
1.7.198
1.7.197
1.7.196
1.7.194
1.7.193
1.7.192
1.7.191
1.7.190
1.7.189
1.7.187
1.7.186
1.7.185
1.7.184
1.7.183
1.7.182
1.7.181
1.7.180
1.7.179
1.7.178
1.7.177
1.7.176
1.7.175
1.7.174
1.7.173
1.7.172
1.7.171
1.7.170
1.7.169
1.7.168
1.7.167
1.7.166
1.7.165
1.7.164
1.7.162
1.7.161
1.7.160
1.7.159
1.7.157
1.7.156
1.7.155
1.7.154
1.7.153
1.7.152
1.7.151
1.7.150
1.7.149
1.7.146
1.7.145
1.7.144
1.7.143
1.7.138
1.7.133
1.7.132
1.7.131
1.7.130
1.7.128
1.7.126
1.7.125
1.7.121
1.7.120
1.7.119
1.7.118
1.7.117
1.7.116
1.7.115
1.7.114
1.7.113
1.7.112
1.7.111
1.7.110
1.7.109
1.7.108
1.7.107
1.7.106
1.7.105
1.7.104
1.7.103
1.7.102
1.7.101
1.7.100
1.7.99
1.7.98
1.7.97
1.7.96
1.7.95
1.7.94
1.7.93
1.7.92
1.7.91
1.7.90
1.7.89
1.7.88
1.7.87
1.7.86
1.7.85
1.7.84
1.7.83
1.7.82
1.7.81
1.7.80
1.7.79
1.7.78
1.7.77
1.7.76
1.7.75
1.7.74
1.7.71
1.7.70
1.7.69
1.7.66
1.7.65
1.7.64
1.7.63
1.7.62
1.7.61
1.7.59
1.7.58
1.7.57
1.7.56
1.7.55
1.7.54
1.7.53
1.7.52
1.7.51
1.7.49
1.7.44
1.7.43
1.7.42
1.7.41
1.7.39
1.7.38
1.7.37
1.7.36
1.7.34
1.7.33
1.7.31
1.7.30
1.7.29
1.7.28
1.7.27
1.7.26
1.7.25
1.7.24
1.7.23
1.7.22
1.7.21
1.7.20
1.7.19
1.7.18
1.7.17
1.7.16
1.7.15
1.7.14
1.7.13
1.7.12
1.7.11
1.7.10
1.7.9
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.20
1.6.19
1.6.18
1.6.17
1.6.16
1.6.15
1.6.14
1.6.13
1.6.12
1.6.11
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.5.2
1.5.1
1.5.0
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.2
1.3.1
1.3.0
1.2.10
1.2.9
1.2.8
1.2.7
1.2.5
1.2.4
1.2.2
1.2.1
1.2.0
1.1.0
1.0.0
A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more
Current section
Files
Jump to
Current section
Files
lib/modules/pages/listing_cache.ex
defmodule PhoenixKit.Modules.Pages.ListingCache do
@moduledoc """
Caches pages group listing metadata to avoid expensive filesystem scans on every request.
Instead of scanning 50+ files per request, the listing page reads a single
`.listing_cache.json` file containing all post metadata.
## How It Works
1. When a post is created/updated/published, `regenerate/1` is called
2. This scans all posts and writes metadata to `.listing_cache.json`
3. `render_blog_listing` reads from cache instead of scanning filesystem
4. Cache includes: title, slug, date, status, languages, versions (no content)
## Cache File Location
priv/pages/{group-slug}/.listing_cache.json
(With legacy fallback to `priv/blogging/{group-slug}/.listing_cache.json`)
## Performance
- Before: ~500ms (50+ file operations)
- After: ~20ms (1 file read + JSON parse)
## Cache Invalidation
Cache is regenerated when:
- Post is created
- Post is updated (metadata or content)
- Post status changes (draft/published/archived)
- Translation is added
- Version is created
## In-Memory Caching with :persistent_term
For sub-millisecond performance, parsed cache data is stored in `:persistent_term`.
- First read after restart: loads from file, parses JSON, stores in :persistent_term (~2ms)
- Subsequent reads: direct memory access (~0.1μs, no variance)
- On regenerate: updates both file and :persistent_term
- On invalidate: clears :persistent_term entry
The JSON file provides persistence across restarts. :persistent_term provides
zero-copy, sub-microsecond reads during runtime.
"""
alias PhoenixKit.Modules.Pages
alias PhoenixKit.Modules.Pages.Storage
alias PhoenixKit.Settings
alias PhoenixKit.Utils.Date, as: UtilsDate
require Logger
# Suppress dialyzer false positive for guard clause in read_from_file_and_cache
@dialyzer {:nowarn_function, read_from_file_and_cache: 3}
@cache_filename ".listing_cache.json"
@persistent_term_prefix :phoenix_kit_pages_listing_cache
@persistent_term_loaded_at_prefix :phoenix_kit_pages_listing_cache_loaded_at
@persistent_term_file_generated_at_prefix :phoenix_kit_pages_listing_cache_file_generated_at
# ETS table for regeneration locks (provides atomic test-and-set via insert_new)
@lock_table :phoenix_kit_pages_listing_cache_locks
# New settings keys (write to these)
@file_cache_key "pages_file_cache_enabled"
@memory_cache_key "pages_memory_cache_enabled"
# Legacy settings keys (read from these as fallback)
@legacy_file_cache_key "blogging_file_cache_enabled"
@legacy_memory_cache_key "blogging_memory_cache_enabled"
@doc """
Reads the cached listing for a pages group.
Returns `{:ok, posts}` if cache exists and is valid.
Returns `{:error, :cache_miss}` if cache doesn't exist, is corrupt, or caching is disabled.
Respects the `pages_file_cache_enabled` and `pages_memory_cache_enabled` settings
(with fallback to legacy `blogging_*` keys).
"""
@spec read(String.t()) :: {:ok, [map()]} | {:error, :cache_miss}
def read(blog_slug) do
memory_enabled = memory_cache_enabled?()
file_enabled = file_cache_enabled?()
term_key = persistent_term_key(blog_slug)
cond do
# Both caches disabled
not memory_enabled and not file_enabled ->
{:error, :cache_miss}
# Memory enabled and found in persistent_term
memory_enabled and memory_cache_hit?(term_key) ->
safe_persistent_term_get(term_key)
# Memory enabled but not found, try file
memory_enabled and file_enabled ->
read_from_file_and_cache(blog_slug, term_key, true)
# Memory enabled, file disabled, not in memory
memory_enabled ->
{:error, :cache_miss}
# Memory disabled, file enabled
file_enabled ->
read_from_file_only(blog_slug)
# Fallback
true ->
{:error, :cache_miss}
end
end
defp memory_cache_hit?(term_key) do
case safe_persistent_term_get(term_key) do
{:ok, _} -> true
:not_found -> false
end
end
# Safely get from :persistent_term (returns :not_found instead of raising)
defp safe_persistent_term_get(key) do
{:ok, :persistent_term.get(key)}
rescue
ArgumentError -> :not_found
end
# Read from JSON file and optionally store in :persistent_term
defp read_from_file_and_cache(blog_slug, term_key, store_in_memory) do
cache_path = cache_path(blog_slug)
with {:ok, content} <- read_cache_file(cache_path),
{:ok, normalized_posts, generated_at} <- parse_cache_content(content, blog_slug) do
if store_in_memory do
store_posts_in_memory(blog_slug, term_key, normalized_posts, generated_at)
end
{:ok, normalized_posts}
end
end
defp read_cache_file(cache_path) do
case File.read(cache_path) do
{:ok, content} -> {:ok, content}
{:error, :enoent} -> {:error, :cache_miss}
{:error, _reason} -> {:error, :cache_miss}
end
end
defp parse_cache_content(content, blog_slug) do
case Jason.decode(content) do
{:ok, %{"posts" => posts} = data} ->
normalized_posts = Enum.map(posts, &normalize_post/1)
generated_at = Map.get(data, "generated_at")
{:ok, normalized_posts, generated_at}
{:ok, _} ->
Logger.warning("[ListingCache] Invalid cache format for #{blog_slug}")
{:error, :cache_miss}
{:error, reason} ->
Logger.warning(
"[ListingCache] Failed to parse cache for #{blog_slug}: #{inspect(reason)}"
)
{:error, :cache_miss}
end
end
defp store_posts_in_memory(blog_slug, term_key, normalized_posts, generated_at) do
safe_persistent_term_put(term_key, normalized_posts)
safe_persistent_term_put(
loaded_at_key(blog_slug),
UtilsDate.utc_now() |> DateTime.to_iso8601()
)
if generated_at do
safe_persistent_term_put(file_generated_at_key(blog_slug), generated_at)
end
Logger.debug(
"[ListingCache] Loaded #{blog_slug} from file into :persistent_term (#{length(normalized_posts)} posts)"
)
end
# Read from JSON file only (no :persistent_term storage)
defp read_from_file_only(blog_slug) do
cache_path = cache_path(blog_slug)
with {:ok, content} <- read_cache_file(cache_path),
{:ok, normalized_posts, _generated_at} <- parse_cache_content(content, blog_slug) do
{:ok, normalized_posts}
end
end
@doc """
Regenerates the listing cache for a blog.
Scans all posts using the standard `list_posts` function and writes
the metadata to `.listing_cache.json`.
This should be called after any post operation that changes the listing:
- create_post
- update_post
- add_language_to_post
- create_new_version
Returns `:ok` on success or `{:error, reason}` on failure.
"""
@spec regenerate(String.t()) :: :ok | {:error, any()}
def regenerate(blog_slug) do
file_enabled = file_cache_enabled?()
memory_enabled = memory_cache_enabled?()
# If both caches are disabled, nothing to do
if not file_enabled and not memory_enabled do
:ok
else
do_regenerate(blog_slug, file_enabled, memory_enabled)
end
rescue
error ->
Logger.error(
"[ListingCache] Failed to regenerate cache for #{blog_slug}: #{inspect(error)}"
)
{:error, {:regenerate_failed, error}}
end
defp do_regenerate(blog_slug, file_enabled, memory_enabled) do
start_time = System.monotonic_time(:millisecond)
# Fetch all posts using the existing storage layer
posts =
case Pages.get_group_mode(blog_slug) do
"slug" -> Storage.list_posts_slug_mode(blog_slug, nil)
_ -> Storage.list_posts(blog_slug, nil)
end
# Convert to cacheable format (strip content, keep metadata)
serialized_posts = Enum.map(posts, &safe_serialize_post/1)
normalized_posts = Enum.map(serialized_posts, &normalize_post/1)
# Generate timestamp once for consistency
generated_at = UtilsDate.utc_now() |> DateTime.to_iso8601()
# Write to file cache if enabled
file_result =
if file_enabled do
cache_data = %{
"generated_at" => generated_at,
"post_count" => length(posts),
"posts" => serialized_posts
}
write_cache_file(cache_path(blog_slug), cache_data)
else
:ok
end
# Update :persistent_term if enabled
if memory_enabled do
safe_persistent_term_put(persistent_term_key(blog_slug), normalized_posts)
safe_persistent_term_put(loaded_at_key(blog_slug), generated_at)
safe_persistent_term_put(file_generated_at_key(blog_slug), generated_at)
end
elapsed = System.monotonic_time(:millisecond) - start_time
case file_result do
:ok ->
Logger.debug(
"[ListingCache] Regenerated cache for #{blog_slug} (#{length(posts)} posts) in #{elapsed}ms"
)
:ok
{:error, reason} = error ->
Logger.error("[ListingCache] Failed to write cache for #{blog_slug}: #{inspect(reason)}")
error
end
end
# Lock timeout in milliseconds (30 seconds)
# If a lock is older than this, it's considered stale (process likely died)
@lock_timeout_ms 30_000
@doc """
Regenerates the cache if no other process is already regenerating it.
This prevents the "thundering herd" problem where multiple concurrent requests
all trigger cache regeneration simultaneously after a server restart.
Uses ETS with `insert_new/2` for atomic lock acquisition - only one process
can acquire the lock at a time. The lock includes a timestamp and will be
considered stale after #{@lock_timeout_ms}ms to prevent permanent lockout
if a process dies mid-regeneration.
Returns:
- `:ok` if regeneration was performed successfully
- `:already_in_progress` if another process is currently regenerating
- `{:error, reason}` if regeneration failed
## Usage
On cache miss in read paths, use this instead of `regenerate/1`:
case ListingCache.regenerate_if_not_in_progress(blog_slug) do
:ok -> # Cache is ready, read from it
:already_in_progress -> # Fall back to filesystem scan
{:error, _} -> # Fall back to filesystem scan
end
"""
@spec regenerate_if_not_in_progress(String.t()) :: :ok | :already_in_progress | {:error, any()}
def regenerate_if_not_in_progress(blog_slug) do
ensure_lock_table_exists()
now = System.monotonic_time(:millisecond)
# Try to atomically acquire the lock using ETS insert_new
# Returns true if inserted (lock acquired), false if key already exists
case :ets.insert_new(@lock_table, {blog_slug, now}) do
true ->
# We acquired the lock - perform regeneration
do_regenerate_with_lock(blog_slug)
false ->
# Lock exists - check if it's stale
handle_existing_lock(blog_slug, now)
end
end
# Handle case where lock already exists - check staleness
defp handle_existing_lock(blog_slug, now) do
case :ets.lookup(@lock_table, blog_slug) do
[{^blog_slug, lock_timestamp}] ->
lock_age = now - lock_timestamp
if lock_age < @lock_timeout_ms do
# Lock is valid and recent - another process is regenerating
Logger.debug(
"[ListingCache] Regeneration already in progress for #{blog_slug} (#{lock_age}ms ago), skipping"
)
:already_in_progress
else
# Lock is stale - previous process likely died
# Try to take over by deleting and re-acquiring atomically
take_over_stale_lock(blog_slug, lock_timestamp, lock_age, now)
end
[] ->
# Lock was released between insert_new and lookup - try again
regenerate_if_not_in_progress(blog_slug)
end
end
# Attempt to take over a stale lock using compare-and-delete
defp take_over_stale_lock(blog_slug, old_timestamp, lock_age, now) do
# Use match_delete for atomic compare-and-delete
# Only deletes if the timestamp matches (no one else took over)
case :ets.select_delete(@lock_table, [{{blog_slug, old_timestamp}, [], [true]}]) do
1 ->
# Successfully deleted stale lock - now try to acquire
Logger.warning(
"[ListingCache] Found stale lock for #{blog_slug} (#{lock_age}ms old), taking over regeneration"
)
case :ets.insert_new(@lock_table, {blog_slug, now}) do
true ->
do_regenerate_with_lock(blog_slug)
false ->
# Another process beat us to it
:already_in_progress
end
0 ->
# Lock was already taken over by another process or timestamp changed
:already_in_progress
end
end
# Perform regeneration while holding the lock
defp do_regenerate_with_lock(blog_slug) do
result = regenerate(blog_slug)
case result do
:ok -> :ok
{:error, _} = error -> error
end
after
# Always release the lock when done (success or failure)
:ets.delete(@lock_table, blog_slug)
end
# Ensure the ETS table for locks exists (lazy initialization)
defp ensure_lock_table_exists do
case :ets.whereis(@lock_table) do
:undefined ->
# Table doesn't exist - create it
# Use :public so any process can read/write
# Use :named_table so we can reference by atom
# Use :set for key-value storage
try do
:ets.new(@lock_table, [:set, :public, :named_table])
rescue
ArgumentError ->
# Table was created by another process between whereis and new
:ok
end
_tid ->
:ok
end
end
# Safely put to :persistent_term (logs warning on failure instead of crashing)
defp safe_persistent_term_put(key, value) do
:persistent_term.put(key, value)
rescue
error ->
Logger.warning("[ListingCache] Failed to write to :persistent_term: #{inspect(error)}")
:error
end
# Safely serialize a post (returns empty map on failure instead of crashing)
defp safe_serialize_post(post) do
serialize_post(post)
rescue
error ->
Logger.warning("[ListingCache] Failed to serialize post: #{inspect(error)}")
%{"slug" => "error", "metadata" => %{"title" => "Error loading post"}}
end
@doc """
Regenerates only the file cache without loading into memory.
This scans all posts and writes to `.listing_cache.json` but does not
update :persistent_term. Use `load_into_memory/1` separately if needed.
"""
@spec regenerate_file_only(String.t()) :: :ok | {:error, any()}
def regenerate_file_only(blog_slug) do
start_time = System.monotonic_time(:millisecond)
# Fetch all posts using the existing storage layer
posts =
case Pages.get_group_mode(blog_slug) do
"slug" -> Storage.list_posts_slug_mode(blog_slug, nil)
_ -> Storage.list_posts(blog_slug, nil)
end
# Convert to cacheable format (use safe version to handle malformed posts)
serialized_posts = Enum.map(posts, &safe_serialize_post/1)
cache_data = %{
"generated_at" => UtilsDate.utc_now() |> DateTime.to_iso8601(),
"post_count" => length(posts),
"posts" => serialized_posts
}
result = write_cache_file(cache_path(blog_slug), cache_data)
elapsed = System.monotonic_time(:millisecond) - start_time
case result do
:ok ->
Logger.debug(
"[ListingCache] Regenerated file cache for #{blog_slug} (#{length(posts)} posts) in #{elapsed}ms"
)
:ok
{:error, reason} = error ->
Logger.error("[ListingCache] Failed to write cache for #{blog_slug}: #{inspect(reason)}")
error
end
rescue
error ->
Logger.error(
"[ListingCache] Failed to regenerate file cache for #{blog_slug}: #{inspect(error)}"
)
{:error, {:regenerate_failed, error}}
end
@doc """
Loads the cache from file into :persistent_term without regenerating the file.
Returns `:ok` if successful, `{:error, :no_file}` if file doesn't exist,
or `{:error, reason}` for other failures.
"""
@spec load_into_memory(String.t()) :: :ok | {:error, any()}
def load_into_memory(blog_slug) do
cache_path = cache_path(blog_slug)
case File.read(cache_path) do
{:ok, content} ->
case Jason.decode(content) do
{:ok, %{"posts" => posts, "generated_at" => generated_at}} ->
normalized_posts = Enum.map(posts, &normalize_post/1)
safe_persistent_term_put(persistent_term_key(blog_slug), normalized_posts)
safe_persistent_term_put(
loaded_at_key(blog_slug),
UtilsDate.utc_now() |> DateTime.to_iso8601()
)
# Store the file's generated_at so we know what version of data is in memory
safe_persistent_term_put(file_generated_at_key(blog_slug), generated_at)
Logger.debug(
"[ListingCache] Loaded #{blog_slug} from file into :persistent_term (#{length(normalized_posts)} posts)"
)
:ok
{:ok, %{"posts" => posts}} ->
# Fallback for files without generated_at
normalized_posts = Enum.map(posts, &normalize_post/1)
safe_persistent_term_put(persistent_term_key(blog_slug), normalized_posts)
safe_persistent_term_put(
loaded_at_key(blog_slug),
UtilsDate.utc_now() |> DateTime.to_iso8601()
)
:ok
{:ok, _} ->
{:error, :invalid_format}
{:error, reason} ->
{:error, reason}
end
{:error, :enoent} ->
{:error, :no_file}
{:error, reason} ->
{:error, reason}
end
end
@doc """
Invalidates (deletes) the cache for a blog.
Clears both the :persistent_term entry and the JSON file.
The next read will return `:cache_miss`, triggering a fallback to
the filesystem scan.
"""
@spec invalidate(String.t()) :: :ok
def invalidate(blog_slug) do
# Clear :persistent_term entries
term_key = persistent_term_key(blog_slug)
try do
:persistent_term.erase(term_key)
rescue
ArgumentError -> :ok
end
try do
:persistent_term.erase(loaded_at_key(blog_slug))
rescue
ArgumentError -> :ok
end
try do
:persistent_term.erase(file_generated_at_key(blog_slug))
rescue
ArgumentError -> :ok
end
# Then delete the file
cache_path = cache_path(blog_slug)
case File.rm(cache_path) do
:ok ->
Logger.debug("[ListingCache] Invalidated cache for #{blog_slug}")
:ok
{:error, :enoent} ->
:ok
{:error, reason} ->
Logger.warning(
"[ListingCache] Failed to delete cache for #{blog_slug}: #{inspect(reason)}"
)
:ok
end
end
@doc """
Checks if a cache exists for a blog (in :persistent_term or file).
"""
@spec exists?(String.t()) :: boolean()
def exists?(blog_slug) do
case safe_persistent_term_get(persistent_term_key(blog_slug)) do
{:ok, _} -> true
:not_found -> cache_path(blog_slug) |> File.exists?()
end
end
@doc """
Finds a post by slug in the cache.
This is useful for single post views where we need metadata (language_statuses,
version_statuses, allow_version_access) without reading multiple files.
Returns `{:ok, cached_post}` if found, `{:error, :not_found}` otherwise.
"""
@spec find_post(String.t(), String.t()) :: {:ok, map()} | {:error, :not_found | :cache_miss}
def find_post(blog_slug, post_slug) do
case read(blog_slug) do
{:ok, posts} ->
case Enum.find(posts, fn p -> p.slug == post_slug end) do
nil -> {:error, :not_found}
post -> {:ok, post}
end
{:error, _} = error ->
error
end
end
@doc """
Finds a post by path pattern in the cache (for timestamp mode).
Matches posts where the path contains the date/time pattern.
Returns `{:ok, cached_post}` if found, `{:error, :not_found}` otherwise.
"""
@spec find_post_by_path(String.t(), String.t(), String.t()) ::
{:ok, map()} | {:error, :not_found | :cache_miss}
def find_post_by_path(blog_slug, date, time) do
case read(blog_slug) do
{:ok, posts} ->
# Match posts using discrete date and time fields (more robust than path string matching)
# Parse the input date string to compare with the cached Date struct
target_date = parse_date_for_lookup(date)
# Normalize time format (handles both "HH:MM" and "HH:MM:SS")
target_time = normalize_time_for_lookup(time)
case Enum.find(posts, fn p ->
dates_match?(p.date, target_date) && times_match?(p.time, target_time)
end) do
nil -> {:error, :not_found}
post -> {:ok, post}
end
{:error, _} = error ->
error
end
end
# Parse date string for lookup comparison
defp parse_date_for_lookup(date_str) when is_binary(date_str) do
case Date.from_iso8601(date_str) do
{:ok, date} -> date
_ -> date_str
end
end
defp parse_date_for_lookup(date), do: date
# Normalize time to "HH:MM" format for comparison
defp normalize_time_for_lookup(time_str) when is_binary(time_str) do
# Take just HH:MM portion
String.slice(time_str, 0, 5)
end
defp normalize_time_for_lookup(time), do: time
# Compare dates - handles both Date structs and strings
defp dates_match?(nil, _), do: false
defp dates_match?(_, nil), do: false
defp dates_match?(%Date{} = cached, %Date{} = target) do
Date.compare(cached, target) == :eq
end
defp dates_match?(%Date{} = cached, target_str) when is_binary(target_str) do
Date.to_iso8601(cached) == target_str
end
defp dates_match?(_, _), do: false
# Compare times - handles Time structs and "HH:MM" strings
defp times_match?(nil, _), do: false
defp times_match?(_, nil), do: false
defp times_match?(%Time{} = cached, target_str) when is_binary(target_str) do
# Format cached time as HH:MM and compare
cached_str = cached |> Time.to_string() |> String.slice(0, 5)
cached_str == target_str
end
defp times_match?(cached_str, target_str)
when is_binary(cached_str) and is_binary(target_str) do
String.slice(cached_str, 0, 5) == String.slice(target_str, 0, 5)
end
defp times_match?(_, _), do: false
@doc """
Finds a post by URL slug for a specific language.
This enables O(1) lookup from URL slug to internal identifier, supporting
per-language URL slugs for SEO-friendly localized URLs.
## Parameters
- `group_slug` - The pages group
- `language` - The language code to search in
- `url_slug` - The URL slug to find
## Returns
- `{:ok, cached_post}` - Found post (includes internal `slug` for file lookup)
- `{:error, :not_found}` - No post with this URL slug for this language
- `{:error, :cache_miss}` - Cache not available
"""
@spec find_by_url_slug(String.t(), String.t(), String.t()) ::
{:ok, map()} | {:error, :not_found | :cache_miss}
def find_by_url_slug(group_slug, language, url_slug) do
case read(group_slug) do
{:ok, posts} -> find_post_by_url_slug(posts, language, url_slug)
{:error, _} -> {:error, :cache_miss}
end
end
defp find_post_by_url_slug(posts, language, url_slug) do
# Search by language_slugs map first
by_language_slug =
Enum.find(posts, &(Map.get(&1.language_slugs || %{}, language) == url_slug))
# Fallback: match by directory slug (backward compatibility)
by_directory_slug = Enum.find(posts, &(&1.slug == url_slug))
case by_language_slug || by_directory_slug do
nil -> {:error, :not_found}
post -> {:ok, post}
end
end
@doc """
Finds a post by a previous URL slug for 301 redirects.
When a URL slug changes, the old slug is stored in `previous_url_slugs`.
This function finds posts that previously used the given URL slug.
## Returns
- `{:ok, cached_post}` - Found post that previously used this slug
- `{:error, :not_found}` - No post with this previous slug
- `{:error, :cache_miss}` - Cache not available
"""
@spec find_by_previous_url_slug(String.t(), String.t(), String.t()) ::
{:ok, map()} | {:error, :not_found | :cache_miss}
def find_by_previous_url_slug(group_slug, language, url_slug) do
case read(group_slug) do
{:ok, posts} -> find_post_by_previous_slug(posts, language, url_slug)
{:error, _} -> {:error, :cache_miss}
end
end
defp find_post_by_previous_slug(posts, language, url_slug) do
case Enum.find(posts, &post_has_previous_slug?(&1, language, url_slug)) do
nil -> {:error, :not_found}
post -> {:ok, post}
end
end
defp post_has_previous_slug?(post, language, url_slug) do
# Check the per-language previous slugs map first
lang_previous_slugs = Map.get(post, :language_previous_slugs) || %{}
previous_for_lang = Map.get(lang_previous_slugs, language) || []
# Fallback: check metadata.previous_url_slugs for backward compatibility
metadata_previous = Map.get(post.metadata || %{}, :previous_url_slugs) || []
url_slug in previous_for_lang or url_slug in metadata_previous
end
@doc """
Returns the cache file path for a pages group.
"""
@spec cache_path(String.t()) :: String.t()
def cache_path(group_slug) do
Path.join(Storage.group_path(group_slug), @cache_filename)
end
@doc """
Returns the :persistent_term key for a pages group's cache.
"""
@spec persistent_term_key(String.t()) :: tuple()
def persistent_term_key(group_slug) do
{@persistent_term_prefix, group_slug}
end
@doc """
Returns the :persistent_term key for tracking when the memory cache was loaded.
"""
@spec loaded_at_key(String.t()) :: tuple()
def loaded_at_key(blog_slug) do
{@persistent_term_loaded_at_prefix, blog_slug}
end
@doc """
Returns when the memory cache was loaded (ISO 8601 string), or nil if not loaded.
"""
@spec memory_loaded_at(String.t()) :: String.t() | nil
def memory_loaded_at(blog_slug) do
case safe_persistent_term_get(loaded_at_key(blog_slug)) do
{:ok, loaded_at} -> loaded_at
:not_found -> nil
end
end
@doc """
Returns the :persistent_term key for tracking the file's generated_at when loaded into memory.
"""
@spec file_generated_at_key(String.t()) :: tuple()
def file_generated_at_key(blog_slug) do
{@persistent_term_file_generated_at_prefix, blog_slug}
end
@doc """
Returns the file's generated_at timestamp that was stored when the memory cache was loaded.
This tells us what version of the file data is currently in memory.
"""
@spec memory_file_generated_at(String.t()) :: String.t() | nil
def memory_file_generated_at(blog_slug) do
case safe_persistent_term_get(file_generated_at_key(blog_slug)) do
{:ok, generated_at} -> generated_at
:not_found -> nil
end
end
@doc """
Returns whether file caching is enabled.
Uses cached settings to avoid database queries on every call.
Checks new key first, falls back to legacy key.
"""
@spec file_cache_enabled?() :: boolean()
def file_cache_enabled? do
case Settings.get_setting_cached(@file_cache_key, nil) do
nil -> Settings.get_setting_cached(@legacy_file_cache_key, "true") == "true"
value -> value == "true"
end
end
@doc """
Returns whether memory caching (:persistent_term) is enabled.
Uses cached settings to avoid database queries on every call.
Checks new key first, falls back to legacy key.
"""
@spec memory_cache_enabled?() :: boolean()
def memory_cache_enabled? do
case Settings.get_setting_cached(@memory_cache_key, nil) do
nil -> Settings.get_setting_cached(@legacy_memory_cache_key, "true") == "true"
value -> value == "true"
end
end
@doc """
Returns a list of posts that need primary_language migration.
This checks all posts in a group and returns those that either:
1. Have no `primary_language` stored (need backfill)
2. Have `primary_language` different from global setting (need migration decision)
"""
@spec posts_needing_primary_language_migration(String.t()) :: [map()]
def posts_needing_primary_language_migration(blog_slug) do
case read(blog_slug) do
{:ok, posts} ->
global_primary = Storage.get_primary_language()
Enum.filter(posts, fn post ->
# Use atom key since normalized posts use atoms
stored_primary = post[:primary_language]
stored_primary == nil or stored_primary != global_primary
end)
{:error, _} ->
# If cache doesn't exist, scan filesystem directly
scan_posts_needing_migration(blog_slug)
end
end
defp scan_posts_needing_migration(blog_slug) do
global_primary = Storage.get_primary_language()
# Try slug mode first, then timestamp mode (list_posts handles timestamp)
posts = Storage.list_posts_slug_mode(blog_slug)
posts =
if posts == [] do
Storage.list_posts(blog_slug)
else
posts
end
posts
|> Enum.filter(fn post ->
stored_primary = Map.get(post[:metadata] || %{}, :primary_language)
stored_primary == nil or stored_primary != global_primary
end)
|> Enum.map(&serialize_post/1)
|> Enum.map(&normalize_post/1)
end
@doc """
Counts posts by primary_language status in a group.
Returns `%{current: n, needs_migration: n, needs_backfill: n}` where:
- `current` - posts with primary_language matching global setting
- `needs_migration` - posts with different primary_language (were created under old setting)
- `needs_backfill` - posts with no primary_language stored (legacy posts)
"""
@spec count_primary_language_status(String.t()) :: map()
def count_primary_language_status(blog_slug) do
case read(blog_slug) do
{:ok, posts} ->
global_primary = Storage.get_primary_language()
Enum.reduce(posts, %{current: 0, needs_migration: 0, needs_backfill: 0}, fn post, acc ->
# Use atom key since normalized posts use atoms
stored_primary = post[:primary_language]
cond do
stored_primary == nil ->
%{acc | needs_backfill: acc.needs_backfill + 1}
stored_primary == global_primary ->
%{acc | current: acc.current + 1}
true ->
%{acc | needs_migration: acc.needs_migration + 1}
end
end)
{:error, _} ->
# If cache doesn't exist, scan filesystem directly
scan_primary_language_status(blog_slug)
end
end
defp scan_primary_language_status(blog_slug) do
global_primary = Storage.get_primary_language()
# Try slug mode first, then timestamp mode (list_posts handles timestamp)
posts = Storage.list_posts_slug_mode(blog_slug)
posts =
if posts == [] do
Storage.list_posts(blog_slug)
else
posts
end
Enum.reduce(posts, %{current: 0, needs_migration: 0, needs_backfill: 0}, fn post, acc ->
stored_primary = Map.get(post[:metadata] || %{}, :primary_language)
cond do
stored_primary == nil ->
%{acc | needs_backfill: acc.needs_backfill + 1}
stored_primary == global_primary ->
%{acc | current: acc.current + 1}
true ->
%{acc | needs_migration: acc.needs_migration + 1}
end
end)
end
# ===========================================================================
# Legacy Structure Migration Status
# ===========================================================================
@doc """
Counts posts by version structure status for a group.
Returns `%{versioned: n, legacy: n}` where:
- `versioned` - posts with v1/, v2/, etc. structure
- `legacy` - posts with flat file structure (need migration)
"""
@spec count_legacy_structure_status(String.t()) :: map()
def count_legacy_structure_status(blog_slug) do
case read(blog_slug) do
{:ok, posts} ->
Enum.reduce(posts, %{versioned: 0, legacy: 0}, fn post, acc ->
if post[:is_legacy_structure] do
%{acc | legacy: acc.legacy + 1}
else
%{acc | versioned: acc.versioned + 1}
end
end)
{:error, _} ->
# If cache doesn't exist, scan filesystem directly
scan_legacy_structure_status(blog_slug)
end
end
@doc """
Returns list of posts that need version structure migration.
"""
@spec posts_needing_version_migration(String.t()) :: [map()]
def posts_needing_version_migration(blog_slug) do
case read(blog_slug) do
{:ok, posts} ->
Enum.filter(posts, & &1[:is_legacy_structure])
{:error, _} ->
# If cache doesn't exist, scan filesystem directly
scan_posts_needing_version_migration(blog_slug)
end
end
defp scan_legacy_structure_status(blog_slug) do
posts = get_posts_for_scan(blog_slug)
Enum.reduce(posts, %{versioned: 0, legacy: 0}, fn post, acc ->
if Map.get(post, :is_legacy_structure, false) do
%{acc | legacy: acc.legacy + 1}
else
%{acc | versioned: acc.versioned + 1}
end
end)
end
defp scan_posts_needing_version_migration(blog_slug) do
posts = get_posts_for_scan(blog_slug)
Enum.filter(posts, &Map.get(&1, :is_legacy_structure, false))
end
defp get_posts_for_scan(blog_slug) do
# Try slug mode first, then timestamp mode
posts = Storage.list_posts_slug_mode(blog_slug)
if posts == [] do
Storage.list_posts(blog_slug)
else
posts
end
end
# Private functions
defp write_cache_file(path, data) do
# Ensure parent directory exists
path |> Path.dirname() |> File.mkdir_p()
# Write to temp file first, then rename (atomic write)
# Use unique suffix to avoid race conditions when multiple processes regenerate simultaneously
unique_id = :erlang.unique_integer([:positive])
tmp_path = "#{path}.tmp.#{unique_id}"
case Jason.encode(data, pretty: true) do
{:ok, json} ->
case File.write(tmp_path, json) do
:ok ->
result = File.rename(tmp_path, path)
# Clean up temp file if rename failed
if result != :ok, do: File.rm(tmp_path)
result
error ->
error
end
{:error, reason} ->
{:error, {:json_encode, reason}}
end
end
defp serialize_post(post) do
# Build both current and previous slugs for all languages
{language_slugs, language_previous_slugs} = build_all_language_slugs(post)
%{
"group" => post[:group],
"slug" => post[:slug],
"url_slug" => post[:url_slug] || post[:slug],
"date" => serialize_date(post[:date]),
"time" => serialize_time(post[:time]),
"path" => post[:path],
"full_path" => post[:full_path],
"mode" => to_string(post[:mode]),
"language" => post[:language],
"available_languages" => post[:available_languages] || [],
"language_statuses" => post[:language_statuses] || %{},
# Per-language URL slugs for SEO-friendly localized URLs
"language_slugs" => language_slugs,
# Per-language previous URL slugs for 301 redirects
"language_previous_slugs" => language_previous_slugs,
"version" => post[:version],
"available_versions" => post[:available_versions] || [],
"version_statuses" => serialize_version_statuses(post[:version_statuses]),
"version_dates" => post[:version_dates] || %{},
"version_languages" => serialize_version_languages(post[:version_languages]),
"is_legacy_structure" => post[:is_legacy_structure] || false,
"metadata" => serialize_metadata(post[:metadata]),
# Pre-compute excerpt for listing page (avoids needing full content)
"excerpt" => extract_excerpt(post[:content], post[:metadata]),
# Primary language for this post (controls versioning/status inheritance)
# NOTE: Do NOT fall back to global setting - we need nil to detect posts needing backfill
"primary_language" => Map.get(post[:metadata] || %{}, :primary_language)
}
end
# Build both language_slugs and language_previous_slugs maps
# Returns {language_slugs, language_previous_slugs}
# language_slugs: language -> current url_slug
# language_previous_slugs: language -> [previous_url_slugs]
defp build_all_language_slugs(post) do
current_lang = post[:language]
current_url_slug = post[:url_slug] || post[:slug]
current_previous = Map.get(post[:metadata] || %{}, :previous_url_slugs) || []
available_langs = post[:available_languages] || []
group_slug = post[:group] || post[:blog]
post_slug = post[:slug]
# Start with the current language's data
base_slugs = %{current_lang => current_url_slug}
base_previous = %{current_lang => current_previous}
# For each available language, read its url_slug and previous_url_slugs
{final_slugs, final_previous} =
Enum.reduce(available_langs, {base_slugs, base_previous}, fn lang, {slugs_acc, prev_acc} ->
if Map.has_key?(slugs_acc, lang) do
{slugs_acc, prev_acc}
else
# Read both url_slug and previous_url_slugs from this language's file
{url_slug, prev_slugs} = get_slugs_for_language(group_slug, post_slug, lang, post)
{Map.put(slugs_acc, lang, url_slug), Map.put(prev_acc, lang, prev_slugs)}
end
end)
{final_slugs, final_previous}
end
# Gets both url_slug and previous_url_slugs for a specific language
# Handles both slug mode and timestamp mode posts correctly
defp get_slugs_for_language(group_slug, post_slug, lang, post) do
# Use appropriate read function based on post mode
result =
case post[:mode] do
:timestamp ->
# For timestamp mode, use the path-based read
post_identifier = extract_timestamp_identifier_for_cache(post[:path])
if post_identifier do
path = Path.join([group_slug, post_identifier, Storage.language_filename(lang)])
Storage.read_post(group_slug, path)
else
{:error, :invalid_path}
end
_ ->
# For slug mode, use the slug-based read
Storage.read_post_slug_mode(group_slug, post_slug, lang, nil)
end
case result do
{:ok, lang_post} ->
url_slug = lang_post.url_slug
previous = Map.get(lang_post.metadata, :previous_url_slugs) || []
{url_slug, previous}
{:error, _} ->
# File doesn't exist or can't be read - use defaults
{post_slug, []}
end
rescue
_ -> {post[:slug] || post_slug, []}
end
# Extract timestamp identifier (date/time) from a timestamp mode path
defp extract_timestamp_identifier_for_cache(path) when is_binary(path) do
case Regex.run(~r/(\d{4}-\d{2}-\d{2}\/\d{2}:\d{2})/, path) do
[_, timestamp] -> timestamp
nil -> nil
end
end
defp extract_timestamp_identifier_for_cache(_), do: nil
# Extract excerpt: use description if available, otherwise extract from content
defp extract_excerpt(_content, %{description: desc}) when is_binary(desc) and desc != "",
do: desc
defp extract_excerpt(content, _metadata) when is_binary(content) do
# Get first paragraph or content before <!-- more --> tag
excerpt_text =
if String.contains?(content, "<!-- more -->") do
content
|> String.split("<!-- more -->")
|> List.first()
|> String.trim()
else
content
|> String.split(~r/\n\n+/)
|> Enum.reject(&String.starts_with?(&1, "#"))
|> List.first()
|> case do
nil -> ""
text -> String.trim(text)
end
end
# Strip markdown formatting and limit length
excerpt_text
|> String.replace(~r/[#*_`\[\]()]/, "")
|> String.replace(~r/\s+/, " ")
|> String.trim()
|> String.slice(0, 300)
end
defp extract_excerpt(_, _), do: nil
defp serialize_metadata(nil), do: %{}
defp serialize_metadata(metadata) when is_map(metadata) do
%{
"title" => Map.get(metadata, :title),
"description" => Map.get(metadata, :description),
"slug" => Map.get(metadata, :slug),
"status" => Map.get(metadata, :status),
"published_at" => Map.get(metadata, :published_at),
"featured_image_uuid" => Map.get(metadata, :featured_image_uuid),
"version" => Map.get(metadata, :version),
"allow_version_access" => Map.get(metadata, :allow_version_access),
"url_slug" => Map.get(metadata, :url_slug),
"previous_url_slugs" => Map.get(metadata, :previous_url_slugs)
}
end
defp serialize_date(nil), do: nil
defp serialize_date(%Date{} = date), do: Date.to_iso8601(date)
defp serialize_date(date) when is_binary(date), do: date
defp serialize_time(nil), do: nil
defp serialize_time(%Time{} = time), do: Time.to_string(time)
defp serialize_time(time) when is_binary(time), do: time
defp serialize_version_statuses(nil), do: %{}
defp serialize_version_statuses(statuses) when is_map(statuses) do
# Convert integer keys to strings for JSON
Map.new(statuses, fn {k, v} -> {to_string(k), v} end)
end
defp serialize_version_languages(nil), do: %{}
defp serialize_version_languages(version_languages) when is_map(version_languages) do
# Convert integer keys to strings for JSON
Map.new(version_languages, fn {k, v} -> {to_string(k), v} end)
end
defp normalize_post(post) when is_map(post) do
slug = post["slug"]
%{
# Support both "group" (new) and "blog" (old cache) keys
group: post["group"] || post["blog"],
slug: slug,
url_slug: post["url_slug"] || slug,
date: parse_date(post["date"]),
time: parse_time(post["time"]),
path: post["path"],
full_path: post["full_path"],
mode: parse_mode(post["mode"]),
language: post["language"],
available_languages: post["available_languages"] || [],
language_statuses: post["language_statuses"] || %{},
# Per-language URL slugs for SEO-friendly localized URLs
language_slugs: post["language_slugs"] || %{},
# Per-language previous URL slugs for 301 redirects
language_previous_slugs: post["language_previous_slugs"] || %{},
version: post["version"],
available_versions: post["available_versions"] || [],
version_statuses: parse_version_statuses(post["version_statuses"]),
version_dates: post["version_dates"] || %{},
version_languages: parse_version_languages(post["version_languages"]),
is_legacy_structure: post["is_legacy_structure"] || false,
metadata: normalize_metadata(post["metadata"]),
# Primary language for this post (controls versioning/status inheritance)
primary_language: post["primary_language"],
# Use pre-computed excerpt as content for template compatibility
# The template's extract_excerpt() will just return this text
content: post["excerpt"]
}
end
defp normalize_metadata(nil), do: %{}
defp normalize_metadata(metadata) when is_map(metadata) do
%{
title: metadata["title"],
description: metadata["description"],
slug: metadata["slug"],
status: metadata["status"],
published_at: metadata["published_at"],
featured_image_uuid: metadata["featured_image_uuid"],
version: metadata["version"],
allow_version_access: metadata["allow_version_access"],
url_slug: metadata["url_slug"],
previous_url_slugs: metadata["previous_url_slugs"]
}
end
defp parse_date(nil), do: nil
defp parse_date(date_str) when is_binary(date_str) do
case Date.from_iso8601(date_str) do
{:ok, date} -> date
_ -> nil
end
end
defp parse_time(nil), do: nil
defp parse_time(time_str) when is_binary(time_str) do
case Time.from_iso8601(time_str) do
{:ok, time} ->
time
# Try parsing without seconds (HH:MM format)
_ ->
case Time.from_iso8601(time_str <> ":00") do
{:ok, time} -> time
_ -> nil
end
end
end
defp parse_mode("slug"), do: :slug
defp parse_mode("timestamp"), do: :timestamp
defp parse_mode(_), do: :timestamp
defp parse_version_statuses(nil), do: %{}
defp parse_version_statuses(statuses) when is_map(statuses) do
# Convert string keys back to integers
Map.new(statuses, fn {k, v} ->
key =
case Integer.parse(k) do
{int, ""} -> int
_ -> k
end
{key, v}
end)
end
defp parse_version_languages(nil), do: %{}
defp parse_version_languages(version_languages) when is_map(version_languages) do
# Convert string keys back to integers
Map.new(version_languages, fn {k, v} ->
key =
case Integer.parse(k) do
{int, ""} -> int
_ -> k
end
{key, v}
end)
end
end