Packages
phoenix_kit
2.7.0
2.10.0
2.9.0
2.8.1
2.8.0
2.7.0
2.6.0
2.5.0
2.4.0
2.3.0
2.2.0
2.1.0
2.0.1
2.0.0
1.7.236
1.7.235
1.7.234
1.7.233
1.7.232
1.7.231
1.7.230
1.7.229
1.7.228
1.7.227
1.7.226
1.7.225
1.7.224
1.7.223
1.7.222
1.7.221
1.7.220
1.7.219
1.7.218
1.7.217
1.7.216
1.7.215
1.7.214
1.7.213
1.7.212
1.7.211
1.7.210
1.7.209
1.7.208
1.7.207
1.7.206
1.7.205
1.7.204
1.7.203
1.7.202
1.7.201
1.7.200
1.7.199
1.7.198
1.7.197
1.7.196
1.7.194
1.7.193
1.7.192
1.7.191
1.7.190
1.7.189
1.7.187
1.7.186
1.7.185
1.7.184
1.7.183
1.7.182
1.7.181
1.7.180
1.7.179
1.7.178
1.7.177
1.7.176
1.7.175
1.7.174
1.7.173
1.7.172
1.7.171
1.7.170
1.7.169
1.7.168
1.7.167
1.7.166
1.7.165
1.7.164
1.7.162
1.7.161
1.7.160
1.7.159
1.7.157
1.7.156
1.7.155
1.7.154
1.7.153
1.7.152
1.7.151
1.7.150
1.7.149
1.7.146
1.7.145
1.7.144
1.7.143
1.7.138
1.7.133
1.7.132
1.7.131
1.7.130
1.7.128
1.7.126
1.7.125
1.7.121
1.7.120
1.7.119
1.7.118
1.7.117
1.7.116
1.7.115
1.7.114
1.7.113
1.7.112
1.7.111
1.7.110
1.7.109
1.7.108
1.7.107
1.7.106
1.7.105
1.7.104
1.7.103
1.7.102
1.7.101
1.7.100
1.7.99
1.7.98
1.7.97
1.7.96
1.7.95
1.7.94
1.7.93
1.7.92
1.7.91
1.7.90
1.7.89
1.7.88
1.7.87
1.7.86
1.7.85
1.7.84
1.7.83
1.7.82
1.7.81
1.7.80
1.7.79
1.7.78
1.7.77
1.7.76
1.7.75
1.7.74
1.7.71
1.7.70
1.7.69
1.7.66
1.7.65
1.7.64
1.7.63
1.7.62
1.7.61
1.7.59
1.7.58
1.7.57
1.7.56
1.7.55
1.7.54
1.7.53
1.7.52
1.7.51
1.7.49
1.7.44
1.7.43
1.7.42
1.7.41
1.7.39
1.7.38
1.7.37
1.7.36
1.7.34
1.7.33
1.7.31
1.7.30
1.7.29
1.7.28
1.7.27
1.7.26
1.7.25
1.7.24
1.7.23
1.7.22
1.7.21
1.7.20
1.7.19
1.7.18
1.7.17
1.7.16
1.7.15
1.7.14
1.7.13
1.7.12
1.7.11
1.7.10
1.7.9
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.20
1.6.19
1.6.18
1.6.17
1.6.16
1.6.15
1.6.14
1.6.13
1.6.12
1.6.11
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.5.2
1.5.1
1.5.0
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.2
1.3.1
1.3.0
1.2.10
1.2.9
1.2.8
1.2.7
1.2.5
1.2.4
1.2.2
1.2.1
1.2.0
1.1.0
1.0.0
A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more
Current section
Files
Jump to
Current section
Files
lib/phoenix_kit_web/live/settings/crawlers.ex
defmodule PhoenixKitWeb.Live.Settings.Crawlers do
@moduledoc """
Crawlers settings LiveView — the bot-policy page.
Sections, top to bottom:
* **Robots directive** — the global noindex/nofollow staging switch.
* **Bot access** — per-group allow/block toggles over the curated registry
(`PhoenixKit.Modules.Crawlers.Bots`).
* **robots.txt** — the complete generated file for copy-paste, built from
the toggles above. PhoenixKit deliberately does **not** write or serve
this file: it is host policy, and `Plug.Static` answers before the
router, so a generated route would be dead code on most installs and a
silent override on the rest.
* **llms.txt** — served (root + prefixed) when the module is enabled;
the operator's extra markdown is edited here.
* **Verification** — Google/Bing site-verification meta contents.
* **Enforcement** — the default-off application-level UA blocker, with its
advisory-grade caveat stated plainly.
"""
use PhoenixKitWeb, :live_view
use Gettext, backend: PhoenixKitWeb.Gettext
alias PhoenixKit.Modules.Crawlers
alias PhoenixKit.Modules.Crawlers.Bots
alias PhoenixKit.Modules.Crawlers.LlmsTxt
alias PhoenixKit.Modules.Crawlers.RobotsTxt
alias PhoenixKit.Settings
alias PhoenixKit.Utils.Routes
def mount(_params, _session, socket) do
if Crawlers.module_enabled?() do
socket =
socket
|> assign(:page_title, gettext("Crawler Settings"))
|> assign(:project_title, Settings.get_project_title())
|> assign(:current_path, get_current_path(socket.assigns.current_locale_base))
# The ROOT urls, deliberately — not `Routes.url/1`, which prefixes
# them. Core serves /sitemap.xml and /llms.txt at both the root and
# under the kit prefix precisely because crawlers look at the root,
# so advertising the prefixed ones would defeat the point.
|> assign(:sitemap_url, Routes.base_url() <> "/sitemap.xml")
|> assign(:llms_txt_url, Routes.base_url() <> "/llms.txt")
|> assign(:robots_txt_present?, File.exists?(robots_txt_path()))
|> assign(:robots_txt_references_sitemap?, robots_txt_references_sitemap?())
|> assign(:groups, Bots.groups())
|> assign_policy_state()
{:ok, socket}
else
socket =
socket
|> put_flash(
:error,
gettext(
"Crawlers module is disabled. Enable it from the Modules page to configure settings."
)
)
|> redirect(to: Routes.path("/admin/modules", locale: socket.assigns.current_locale_base))
{:ok, socket}
end
end
def handle_event("toggle_no_index", _params, socket) do
new_value = !socket.assigns.no_index_enabled
result =
if new_value do
Crawlers.enable_no_index()
else
Crawlers.disable_no_index()
end
case result do
{:ok, _setting} ->
message =
if new_value do
gettext("Noindex/nofollow enabled. Search engines will not index this site.")
else
gettext("Noindex/nofollow disabled. Site can be indexed again.")
end
{:noreply,
socket
|> assign(:no_index_enabled, new_value)
|> put_flash(:info, message)}
{:error, _changeset} ->
{:noreply, put_flash(socket, :error, gettext("Failed to update crawler settings"))}
end
end
def handle_event("toggle_group", %{"group" => group_key}, socket) do
# Whitelist against the registry — never build an atom or a settings key
# from raw params.
case Bots.group(group_key) do
nil ->
{:noreply, socket}
group ->
new_value = !Crawlers.group_allowed?(group.key)
case Crawlers.update_group_allowed(group.key, new_value) do
{:ok, _} ->
{:noreply, assign_policy_state(socket)}
{:error, _} ->
{:noreply, put_flash(socket, :error, gettext("Failed to update crawler settings"))}
end
end
end
def handle_event("toggle_block_at_app", _params, socket) do
case Crawlers.update_block_at_app(!socket.assigns.block_at_app) do
{:ok, _} ->
{:noreply, assign_policy_state(socket)}
{:error, _} ->
{:noreply, put_flash(socket, :error, gettext("Failed to update crawler settings"))}
end
end
def handle_event("save_verifications", params, socket) do
google = Map.get(params, "google", "")
bing = Map.get(params, "bing", "")
case Crawlers.update_verifications(google, bing) do
:ok ->
{:noreply,
socket
|> assign_policy_state()
|> put_flash(:info, gettext("Verification tags saved."))}
{:error, _} ->
{:noreply, put_flash(socket, :error, gettext("Failed to update crawler settings"))}
end
end
def handle_event("save_llms_txt", params, socket) do
case Crawlers.update_llms_txt_extra(Map.get(params, "extra", "")) do
{:ok, _} ->
{:noreply,
socket
|> assign_policy_state()
|> put_flash(:info, gettext("llms.txt content saved."))}
{:error, _} ->
{:noreply, put_flash(socket, :error, gettext("Failed to update crawler settings"))}
end
end
# One reader for every piece of policy state the page renders, so all
# events converge on the same refresh and cannot drift from the DB.
defp assign_policy_state(socket) do
allowed = Map.new(Bots.group_keys(), &{&1, Crawlers.group_allowed?(&1)})
socket
|> assign(:no_index_enabled, Crawlers.no_index_enabled?())
|> assign(:group_allowed, allowed)
|> assign(:block_at_app, Crawlers.block_at_app_enabled?())
|> assign(:google_verification, Crawlers.google_verification())
|> assign(:bing_verification, Crawlers.bing_verification())
|> assign(:llms_txt_extra, Crawlers.llms_txt_extra())
|> assign(:llms_txt_preview, LlmsTxt.build())
|> assign(
:robots_txt,
RobotsTxt.build(sitemap_url: Routes.base_url() <> "/sitemap.xml")
)
end
defp get_current_path(locale) do
Routes.path("/admin/settings/crawlers", locale: locale)
end
# Where Plug.Static serves it from in a stock Phoenix app. Read-only: this
# page reports what is there, it never writes the host's robots.txt.
defp robots_txt_path, do: Path.join(["priv", "static", "robots.txt"])
defp robots_txt_references_sitemap? do
case File.read(robots_txt_path()) do
{:ok, contents} -> contents =~ ~r/^\s*sitemap:/im
{:error, _} -> false
end
end
end