Packages
phoenix_kit
2.8.0
2.10.0
2.9.0
2.8.1
2.8.0
2.7.0
2.6.0
2.5.0
2.4.0
2.3.0
2.2.0
2.1.0
2.0.1
2.0.0
1.7.236
1.7.235
1.7.234
1.7.233
1.7.232
1.7.231
1.7.230
1.7.229
1.7.228
1.7.227
1.7.226
1.7.225
1.7.224
1.7.223
1.7.222
1.7.221
1.7.220
1.7.219
1.7.218
1.7.217
1.7.216
1.7.215
1.7.214
1.7.213
1.7.212
1.7.211
1.7.210
1.7.209
1.7.208
1.7.207
1.7.206
1.7.205
1.7.204
1.7.203
1.7.202
1.7.201
1.7.200
1.7.199
1.7.198
1.7.197
1.7.196
1.7.194
1.7.193
1.7.192
1.7.191
1.7.190
1.7.189
1.7.187
1.7.186
1.7.185
1.7.184
1.7.183
1.7.182
1.7.181
1.7.180
1.7.179
1.7.178
1.7.177
1.7.176
1.7.175
1.7.174
1.7.173
1.7.172
1.7.171
1.7.170
1.7.169
1.7.168
1.7.167
1.7.166
1.7.165
1.7.164
1.7.162
1.7.161
1.7.160
1.7.159
1.7.157
1.7.156
1.7.155
1.7.154
1.7.153
1.7.152
1.7.151
1.7.150
1.7.149
1.7.146
1.7.145
1.7.144
1.7.143
1.7.138
1.7.133
1.7.132
1.7.131
1.7.130
1.7.128
1.7.126
1.7.125
1.7.121
1.7.120
1.7.119
1.7.118
1.7.117
1.7.116
1.7.115
1.7.114
1.7.113
1.7.112
1.7.111
1.7.110
1.7.109
1.7.108
1.7.107
1.7.106
1.7.105
1.7.104
1.7.103
1.7.102
1.7.101
1.7.100
1.7.99
1.7.98
1.7.97
1.7.96
1.7.95
1.7.94
1.7.93
1.7.92
1.7.91
1.7.90
1.7.89
1.7.88
1.7.87
1.7.86
1.7.85
1.7.84
1.7.83
1.7.82
1.7.81
1.7.80
1.7.79
1.7.78
1.7.77
1.7.76
1.7.75
1.7.74
1.7.71
1.7.70
1.7.69
1.7.66
1.7.65
1.7.64
1.7.63
1.7.62
1.7.61
1.7.59
1.7.58
1.7.57
1.7.56
1.7.55
1.7.54
1.7.53
1.7.52
1.7.51
1.7.49
1.7.44
1.7.43
1.7.42
1.7.41
1.7.39
1.7.38
1.7.37
1.7.36
1.7.34
1.7.33
1.7.31
1.7.30
1.7.29
1.7.28
1.7.27
1.7.26
1.7.25
1.7.24
1.7.23
1.7.22
1.7.21
1.7.20
1.7.19
1.7.18
1.7.17
1.7.16
1.7.15
1.7.14
1.7.13
1.7.12
1.7.11
1.7.10
1.7.9
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.20
1.6.19
1.6.18
1.6.17
1.6.16
1.6.15
1.6.14
1.6.13
1.6.12
1.6.11
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.5.2
1.5.1
1.5.0
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.2
1.3.1
1.3.0
1.2.10
1.2.9
1.2.8
1.2.7
1.2.5
1.2.4
1.2.2
1.2.1
1.2.0
1.1.0
1.0.0
A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more
Current section
Files
Jump to
Current section
Files
lib/phoenix_kit_web/live/settings/crawlers.html.heex
<PhoenixKitWeb.Components.LayoutWrapper.app_layout
socket={@socket}
flash={@flash}
phoenix_kit_current_scope={assigns[:phoenix_kit_current_scope]}
page_title={gettext("Crawler Settings")}
page_subtitle={
gettext(
"Decide who may read this site — indexing directives, per-bot access, and what crawlers and AI assistants are told."
)
}
current_path={@current_path}
project_title={@project_title}
current_locale={@current_locale}
>
<div class="px-4 py-6 space-y-6">
<div class="card bg-base-100 shadow-xl border border-base-200">
<div class="card-body space-y-4">
<div class="flex items-start justify-between gap-4">
<div>
<p class="text-sm uppercase tracking-wide text-primary font-semibold">
{gettext("Robots Directive")}
</p>
<h2 class="text-2xl font-bold">{gettext("Noindex · Nofollow")}</h2>
<p class="text-base text-base-content/70">
{gettext("Adds")}
<code class="font-mono text-sm bg-base-200 rounded px-1.5 py-0.5">
<meta name="robots" content="noindex,nofollow">
</code>
{gettext(
"to every PhoenixKit layout, blocking crawlers from indexing or following links."
)}
</p>
</div>
<label class="flex items-center gap-2 cursor-pointer">
<span class="text-sm font-semibold text-base-content/70">
{if @no_index_enabled, do: gettext("Enabled"), else: gettext("Disabled")}
</span>
<input
type="checkbox"
class="toggle toggle-primary toggle-lg"
checked={@no_index_enabled}
phx-click="toggle_no_index"
/>
</label>
</div>
<div class={[
"alert",
if(@no_index_enabled, do: "alert-warning", else: "alert-info"),
"bg-base-200/70"
]}>
<.icon
name={
if @no_index_enabled, do: "hero-no-symbol", else: "hero-magnifying-glass-circle"
}
class="w-6 h-6"
/>
<div class="text-sm leading-relaxed">
<%= if @no_index_enabled do %>
{gettext(
"Search engines are instructed to skip this environment. Remove this before launch."
)}
<% else %>
{gettext(
"Crawlers are allowed to index the site. Enable the toggle to hide staging deployments."
)}
<% end %>
</div>
</div>
</div>
</div>
<div class="card bg-base-100 shadow-xl border border-base-200">
<div class="card-body space-y-4">
<div>
<p class="text-sm uppercase tracking-wide text-primary font-semibold">
{gettext("Bot Access")}
</p>
<h2 class="text-2xl font-bold">{gettext("Who may crawl this site")}</h2>
<p class="text-base text-base-content/70">
{gettext(
"Grouped by why they visit. Blocking a group writes it into the generated robots.txt below; everything defaults to allowed."
)}
</p>
</div>
<div class="divide-y divide-base-200">
<div :for={group <- @groups} class="py-3 flex items-start justify-between gap-4">
<div>
<p class="font-semibold">{group.label}</p>
<p class="text-sm text-base-content/60">{group.description}</p>
<p class="text-xs text-base-content/50 font-mono mt-1">
{Enum.map_join(group.bots, " · ", & &1.token)}
</p>
</div>
<label class="flex items-center gap-2 cursor-pointer shrink-0">
<span class="text-sm font-semibold text-base-content/70">
{if @group_allowed[group.key], do: gettext("Allowed"), else: gettext("Blocked")}
</span>
<input
type="checkbox"
class="toggle toggle-primary"
checked={@group_allowed[group.key]}
phx-click="toggle_group"
phx-value-group={group.key}
/>
</label>
</div>
</div>
</div>
</div>
<%!-- robots.txt: generated for copy-paste, never written. See the
LiveView moduledoc for why PhoenixKit does not serve this file. --%>
<div class="card bg-base-100 shadow-xl border border-base-200">
<div class="card-body space-y-4">
<div>
<p class="text-sm uppercase tracking-wide text-primary font-semibold">
{gettext("Crawler Instructions")}
</p>
<h2 class="text-2xl font-bold">robots.txt</h2>
<p class="text-base text-base-content/70">
{gettext(
"PhoenixKit does not serve this file — it is yours, served from priv/static before any route is consulted. This is the complete file your toggles above describe: paste it into priv/static/robots.txt."
)}
</p>
</div>
<div class="relative">
<pre
id="pk-robots-txt-output"
class="font-mono text-sm bg-base-200 rounded p-3 overflow-x-auto whitespace-pre"
>{@robots_txt}</pre>
<button
type="button"
class="btn btn-sm btn-outline absolute top-2 right-2"
phx-hook="CopyToClipboard"
id="pk-robots-txt-copy"
data-copy-target="#pk-robots-txt-output"
>
<.icon name="hero-clipboard-document" class="w-4 h-4" />
<span data-copy-idle>{gettext("Copy")}</span>
<span data-copy-feedback class="hidden">{gettext("Copied!")}</span>
</button>
</div>
<div class={[
"alert bg-base-200/70",
if(@robots_txt_present? and @robots_txt_references_sitemap?,
do: "alert-success",
else: "alert-info"
)
]}>
<.icon
name={
if @robots_txt_present? and @robots_txt_references_sitemap?,
do: "hero-check-circle",
else: "hero-information-circle"
}
class="w-6 h-6"
/>
<div class="text-sm leading-relaxed">
<%= cond do %>
<% not @robots_txt_present? -> %>
{gettext(
"No priv/static/robots.txt found. Without one, crawlers assume everything is allowed — and any groups you blocked above are not being told."
)}
<% not @robots_txt_references_sitemap? -> %>
{gettext(
"priv/static/robots.txt exists but has no Sitemap: line. Paste the file above so crawlers are told where to look."
)}
<% true -> %>
{gettext("priv/static/robots.txt points crawlers at your sitemap.")}
<% end %>
</div>
</div>
<p class="text-xs text-base-content/60">
{gettext(
"The noindex toggle and robots.txt are separate mechanisms: noindex is a per-page meta tag that tells crawlers not to index what they fetched, while robots.txt tells them what not to fetch at all. A staging site usually wants both."
)}
</p>
</div>
</div>
<div class="card bg-base-100 shadow-xl border border-base-200">
<div class="card-body space-y-4">
<div>
<p class="text-sm uppercase tracking-wide text-primary font-semibold">
{gettext("AI Discovery")}
</p>
<h2 class="text-2xl font-bold">llms.txt</h2>
<p class="text-base text-base-content/70">
{gettext(
"The other half of bot policy: a markdown summary telling AI assistants what this site is about, served at"
)}
<a href={@llms_txt_url} target="_blank" class="link link-primary font-mono text-sm">
{@llms_txt_url}
</a>
{gettext(
"while this module is enabled. The heading comes from your project title; everything below it is yours."
)}
</p>
</div>
<form phx-submit="save_llms_txt" id="pk-llms-txt-form" class="space-y-3">
<textarea
name="extra"
rows="6"
class="textarea textarea-bordered w-full font-mono text-sm"
placeholder={
gettext("Optional markdown: a summary paragraph, key links, docs pointers…")
}
>{@llms_txt_extra}</textarea>
<div class="flex items-center justify-between gap-4">
<p class="text-xs text-base-content/60">
{gettext("Preview of the served file:")}
</p>
<button type="submit" class="btn btn-primary btn-sm">{gettext("Save")}</button>
</div>
</form>
<pre class="font-mono text-xs bg-base-200 rounded p-3 overflow-x-auto whitespace-pre">{@llms_txt_preview}</pre>
</div>
</div>
<div class="card bg-base-100 shadow-xl border border-base-200">
<div class="card-body space-y-4">
<div>
<p class="text-sm uppercase tracking-wide text-primary font-semibold">
{gettext("Verification")}
</p>
<h2 class="text-2xl font-bold">{gettext("Search engine verification")}</h2>
<p class="text-base text-base-content/70">
{gettext(
"Paste the verification content Google Search Console or Bing Webmaster Tools gives you — the meta tags are added to every layout head. Pasting the whole <meta> tag also works; the content value is extracted."
)}
</p>
</div>
<form phx-submit="save_verifications" id="pk-verifications-form" class="space-y-3">
<label class="form-control w-full">
<span class="label-text font-medium mb-1">
Google <span class="font-mono text-xs">(google-site-verification)</span>
</span>
<input
type="text"
name="google"
value={@google_verification}
class="input input-bordered w-full font-mono text-sm"
autocomplete="off"
/>
</label>
<label class="form-control w-full">
<span class="label-text font-medium mb-1">
Bing <span class="font-mono text-xs">(msvalidate.01)</span>
</span>
<input
type="text"
name="bing"
value={@bing_verification}
class="input input-bordered w-full font-mono text-sm"
autocomplete="off"
/>
</label>
<div class="flex justify-end">
<button type="submit" class="btn btn-primary btn-sm">{gettext("Save")}</button>
</div>
</form>
</div>
</div>
<div class="card bg-base-100 shadow-xl border border-base-200">
<div class="card-body space-y-4">
<div class="flex items-start justify-between gap-4">
<div>
<p class="text-sm uppercase tracking-wide text-primary font-semibold">
{gettext("Enforcement")}
</p>
<h2 class="text-2xl font-bold">{gettext("Block at the application")}</h2>
<p class="text-base text-base-content/70">
{gettext(
"Answer 403 to requests whose User-Agent matches a blocked group, for bots that ignore robots.txt."
)}
</p>
</div>
<label class="flex items-center gap-2 cursor-pointer shrink-0">
<span class="text-sm font-semibold text-base-content/70">
{if @block_at_app, do: gettext("On"), else: gettext("Off")}
</span>
<input
type="checkbox"
class="toggle toggle-primary toggle-lg"
checked={@block_at_app}
phx-click="toggle_block_at_app"
/>
</label>
</div>
<div class="alert alert-info bg-base-200/70">
<.icon name="hero-information-circle" class="w-6 h-6" />
<div class="text-sm leading-relaxed">
{gettext(
"Advisory-grade: anything can send any User-Agent, so a scraper that lies passes through. Real enforcement belongs at your CDN or reverse proxy — this catches the honest-but-ignorant. Bots identified only by robots.txt tokens (Google-Extended, Applebot-Extended) crawl under their main user agent and are never matched here, so blocking AI training cannot 403 Googlebot."
)}
</div>
</div>
</div>
</div>
</div>
</PhoenixKitWeb.Components.LayoutWrapper.app_layout>