Packages
phoenix_kit
1.7.55
1.7.208
1.7.207
1.7.206
1.7.205
1.7.204
1.7.203
1.7.202
1.7.201
1.7.200
1.7.199
1.7.198
1.7.197
1.7.196
1.7.194
1.7.193
1.7.192
1.7.191
1.7.190
1.7.189
1.7.187
1.7.186
1.7.185
1.7.184
1.7.183
1.7.182
1.7.181
1.7.180
1.7.179
1.7.178
1.7.177
1.7.176
1.7.175
1.7.174
1.7.173
1.7.172
1.7.171
1.7.170
1.7.169
1.7.168
1.7.167
1.7.166
1.7.165
1.7.164
1.7.162
1.7.161
1.7.160
1.7.159
1.7.157
1.7.156
1.7.155
1.7.154
1.7.153
1.7.152
1.7.151
1.7.150
1.7.149
1.7.146
1.7.145
1.7.144
1.7.143
1.7.138
1.7.133
1.7.132
1.7.131
1.7.130
1.7.128
1.7.126
1.7.125
1.7.121
1.7.120
1.7.119
1.7.118
1.7.117
1.7.116
1.7.115
1.7.114
1.7.113
1.7.112
1.7.111
1.7.110
1.7.109
1.7.108
1.7.107
1.7.106
1.7.105
1.7.104
1.7.103
1.7.102
1.7.101
1.7.100
1.7.99
1.7.98
1.7.97
1.7.96
1.7.95
1.7.94
1.7.93
1.7.92
1.7.91
1.7.90
1.7.89
1.7.88
1.7.87
1.7.86
1.7.85
1.7.84
1.7.83
1.7.82
1.7.81
1.7.80
1.7.79
1.7.78
1.7.77
1.7.76
1.7.75
1.7.74
1.7.71
1.7.70
1.7.69
1.7.66
1.7.65
1.7.64
1.7.63
1.7.62
1.7.61
1.7.59
1.7.58
1.7.57
1.7.56
1.7.55
1.7.54
1.7.53
1.7.52
1.7.51
1.7.49
1.7.44
1.7.43
1.7.42
1.7.41
1.7.39
1.7.38
1.7.37
1.7.36
1.7.34
1.7.33
1.7.31
1.7.30
1.7.29
1.7.28
1.7.27
1.7.26
1.7.25
1.7.24
1.7.23
1.7.22
1.7.21
1.7.20
1.7.19
1.7.18
1.7.17
1.7.16
1.7.15
1.7.14
1.7.13
1.7.12
1.7.11
1.7.10
1.7.9
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.20
1.6.19
1.6.18
1.6.17
1.6.16
1.6.15
1.6.14
1.6.13
1.6.12
1.6.11
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.5.2
1.5.1
1.5.0
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.2
1.3.1
1.3.0
1.2.10
1.2.9
1.2.8
1.2.7
1.2.5
1.2.4
1.2.2
1.2.1
1.2.0
1.1.0
1.0.0
A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more
Current section
Files
Jump to
Current section
Files
lib/modules/publishing/workers/validate_migration_worker.ex
defmodule PhoenixKit.Modules.Publishing.Workers.ValidateMigrationWorker do
@moduledoc """
Oban worker that validates filesystem vs database consistency after migration.
Compares post counts and content hashes between filesystem and database
for each publishing group. Reports discrepancies via Logger and PubSub.
Run this BEFORE switching reads to DB (`publishing_storage: :db`).
## Usage
ValidateMigrationWorker.enqueue()
## PubSub Events
Broadcasts to the publishing groups topic:
- `{:migration_validation_completed, results}`
"""
use Oban.Worker, queue: :default, max_attempts: 1
require Logger
alias PhoenixKit.Modules.Publishing
alias PhoenixKit.Modules.Publishing.DBStorage
alias PhoenixKit.Modules.Publishing.PubSub, as: PublishingPubSub
alias PhoenixKit.Modules.Publishing.Storage
alias PhoenixKit.PubSub.Manager
@impl Oban.Worker
def perform(%Oban.Job{}) do
Logger.info("[ValidateMigration] Starting filesystem vs database comparison")
groups = Publishing.list_groups()
results =
Enum.map(groups, fn group ->
slug = group["slug"]
mode = group["mode"] || "timestamp"
validate_group(slug, mode)
end)
total_discrepancies = Enum.sum(Enum.map(results, & &1.discrepancies))
Logger.info(
"[ValidateMigration] Validation complete: " <>
"#{length(results)} groups checked, #{total_discrepancies} discrepancies"
)
broadcast_validation_completed(results)
if total_discrepancies > 0 do
Logger.warning(
"[ValidateMigration] Found #{total_discrepancies} discrepancies — review before switching to DB"
)
end
:ok
end
@impl Oban.Worker
def timeout(_job), do: :timer.minutes(15)
defp validate_group(group_slug, mode) do
# Count filesystem posts
fs_posts =
case mode do
"slug" -> Storage.list_posts_slug_mode(group_slug)
_ -> Storage.list_posts(group_slug)
end
fs_count = length(fs_posts)
# Count database posts
db_posts = DBStorage.list_posts(group_slug)
db_count = length(db_posts)
# Check for DB group existence
db_group = DBStorage.get_group_by_slug(group_slug)
discrepancies =
cond do
is_nil(db_group) ->
Logger.warning("[ValidateMigration] Group #{group_slug}: missing from database")
1
fs_count != db_count ->
Logger.warning(
"[ValidateMigration] Group #{group_slug}: " <>
"FS has #{fs_count} posts, DB has #{db_count} posts"
)
abs(fs_count - db_count)
true ->
# Counts match — spot-check a few posts for content hash
check_content_hashes(group_slug, fs_posts, mode)
end
%{
group: group_slug,
fs_posts: fs_count,
db_posts: db_count,
discrepancies: discrepancies,
group_exists_in_db: not is_nil(db_group)
}
end
defp check_content_hashes(group_slug, fs_posts, mode) do
# Sample up to 10 posts for content hash comparison
sample = Enum.take(fs_posts, 10)
Enum.reduce(sample, 0, fn post, count ->
post_slug = post[:slug]
fs_result =
case mode do
"slug" -> Storage.read_post_slug_mode(group_slug, post_slug)
_ -> Storage.read_post(group_slug, post[:path])
end
db_result = DBStorage.read_post(group_slug, post_slug)
case {fs_result, db_result} do
{{:ok, fs_post}, {:ok, db_post}} ->
fs_hash = content_hash(fs_post[:content])
db_hash = content_hash(db_post[:content])
if fs_hash != db_hash do
Logger.warning(
"[ValidateMigration] Content mismatch: #{group_slug}/#{post_slug} " <>
"(FS hash: #{fs_hash}, DB hash: #{db_hash})"
)
count + 1
else
count
end
{{:ok, _}, {:error, _}} ->
Logger.warning(
"[ValidateMigration] Post #{group_slug}/#{post_slug}: in FS but not in DB"
)
count + 1
_ ->
count
end
end)
end
defp content_hash(nil), do: "nil"
defp content_hash(content) when is_binary(content) do
:crypto.hash(:md5, content)
|> Base.encode16(case: :lower)
|> String.slice(0..7)
end
defp broadcast_validation_completed(results) do
Manager.broadcast(
PublishingPubSub.groups_topic(),
{:migration_validation_completed, results}
)
rescue
_ -> :ok
end
@doc "Creates a validation job."
def create_job do
new(%{"type" => "validate_migration"})
end
@doc "Enqueues a validation job."
def enqueue do
create_job()
|> Oban.insert()
end
end