Packages
phoenix_kit
1.7.92
1.7.208
1.7.207
1.7.206
1.7.205
1.7.204
1.7.203
1.7.202
1.7.201
1.7.200
1.7.199
1.7.198
1.7.197
1.7.196
1.7.194
1.7.193
1.7.192
1.7.191
1.7.190
1.7.189
1.7.187
1.7.186
1.7.185
1.7.184
1.7.183
1.7.182
1.7.181
1.7.180
1.7.179
1.7.178
1.7.177
1.7.176
1.7.175
1.7.174
1.7.173
1.7.172
1.7.171
1.7.170
1.7.169
1.7.168
1.7.167
1.7.166
1.7.165
1.7.164
1.7.162
1.7.161
1.7.160
1.7.159
1.7.157
1.7.156
1.7.155
1.7.154
1.7.153
1.7.152
1.7.151
1.7.150
1.7.149
1.7.146
1.7.145
1.7.144
1.7.143
1.7.138
1.7.133
1.7.132
1.7.131
1.7.130
1.7.128
1.7.126
1.7.125
1.7.121
1.7.120
1.7.119
1.7.118
1.7.117
1.7.116
1.7.115
1.7.114
1.7.113
1.7.112
1.7.111
1.7.110
1.7.109
1.7.108
1.7.107
1.7.106
1.7.105
1.7.104
1.7.103
1.7.102
1.7.101
1.7.100
1.7.99
1.7.98
1.7.97
1.7.96
1.7.95
1.7.94
1.7.93
1.7.92
1.7.91
1.7.90
1.7.89
1.7.88
1.7.87
1.7.86
1.7.85
1.7.84
1.7.83
1.7.82
1.7.81
1.7.80
1.7.79
1.7.78
1.7.77
1.7.76
1.7.75
1.7.74
1.7.71
1.7.70
1.7.69
1.7.66
1.7.65
1.7.64
1.7.63
1.7.62
1.7.61
1.7.59
1.7.58
1.7.57
1.7.56
1.7.55
1.7.54
1.7.53
1.7.52
1.7.51
1.7.49
1.7.44
1.7.43
1.7.42
1.7.41
1.7.39
1.7.38
1.7.37
1.7.36
1.7.34
1.7.33
1.7.31
1.7.30
1.7.29
1.7.28
1.7.27
1.7.26
1.7.25
1.7.24
1.7.23
1.7.22
1.7.21
1.7.20
1.7.19
1.7.18
1.7.17
1.7.16
1.7.15
1.7.14
1.7.13
1.7.12
1.7.11
1.7.10
1.7.9
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.20
1.6.19
1.6.18
1.6.17
1.6.16
1.6.15
1.6.14
1.6.13
1.6.12
1.6.11
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.5.2
1.5.1
1.5.0
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.2
1.3.1
1.3.0
1.2.10
1.2.9
1.2.8
1.2.7
1.2.5
1.2.4
1.2.2
1.2.1
1.2.0
1.1.0
1.0.0
A foundation for building Elixir Phoenix apps — SaaS, social networks, ERP systems, marketplaces, and more
Current section
Files
Jump to
Current section
Files
lib/modules/llm_text/generator.ex
defmodule PhoenixKit.Modules.LLMText.Generator do
@moduledoc """
Generator for LLM-friendly text files.
Produces:
- `llms.txt` - Index file linking to all LLM-readable pages
- Individual page `.md` files per source
## Usage
# Regenerate all sources and rebuild index
Generator.run_all()
# Regenerate a single source and rebuild index
Generator.run_source(MyApp.LLMText.BlogSource)
# Only rebuild the index from all sources
Generator.rebuild_index()
"""
require Logger
alias PhoenixKit.Modules.LLMText.FileStorage
alias PhoenixKit.Modules.LLMText.Sources.Source
@doc """
Regenerates files for one source and rebuilds the llms.txt index.
"""
@spec run_source(module()) :: :ok
def run_source(source_module) do
Logger.info("LLMText.Generator: Running source #{inspect(source_module)}")
files = Source.safe_collect_page_files(source_module)
Enum.each(files, fn {path, content} ->
case FileStorage.write(path, content) do
:ok ->
:ok
{:error, reason} ->
Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}")
end
end)
rebuild_index()
end
@doc """
Regenerates all sources and rebuilds the llms.txt index.
"""
@spec run_all() :: :ok
def run_all do
sources = get_sources()
Logger.info("LLMText.Generator: Running all #{length(sources)} sources")
Enum.each(sources, fn source_module ->
files = Source.safe_collect_page_files(source_module)
Enum.each(files, fn {path, content} ->
case FileStorage.write(path, content) do
:ok ->
:ok
{:error, reason} ->
Logger.warning("LLMText.Generator: Failed to write #{path}: #{inspect(reason)}")
end
end)
end)
rebuild_index()
end
@doc """
Rebuilds the llms.txt index from all sources without regenerating page files.
"""
@spec rebuild_index() :: :ok
def rebuild_index do
sources = get_sources()
Logger.debug("LLMText.Generator: Rebuilding index from #{length(sources)} sources")
entries =
sources
|> Enum.flat_map(&Source.safe_collect_index_entries/1)
content = build_index_content(entries)
case FileStorage.write_index(content) do
:ok ->
:ok
{:error, reason} ->
Logger.warning("LLMText.Generator: Failed to write index: #{inspect(reason)}")
:ok
end
end
@doc """
Builds the llms.txt markdown content from a list of index entries.
Groups entries by their `:group` field. Group order follows first-seen order.
Within each group, entries appear in the order they were provided.
"""
@spec build_index_content([Source.index_entry()]) :: String.t()
def build_index_content(entries) do
site_name = get_site_name()
site_description = get_site_description()
# Build ordered groups (first-seen order) using prepend + reverse for O(n) performance
{groups_reversed, groups_map} =
Enum.reduce(entries, {[], %{}}, fn entry, {order, map} ->
group = Map.get(entry, :group, "General")
if Map.has_key?(map, group) do
{order, Map.update!(map, group, &[entry | &1])}
else
{[group | order], Map.put(map, group, [entry])}
end
end)
groups_ordered = Enum.reverse(groups_reversed)
header =
if site_description && site_description != "" do
"# #{site_name}\n\n> #{site_description}\n\n"
else
"# #{site_name}\n\n"
end
sections =
Enum.map_join(groups_ordered, "\n\n", fn group ->
group_entries = Map.get(groups_map, group, []) |> Enum.reverse()
links =
Enum.map_join(group_entries, "\n", fn entry ->
title = Map.get(entry, :title, "")
url = Map.get(entry, :url, "")
description = Map.get(entry, :description, "")
if description && description != "" do
"- [#{title}](#{url}): #{description}"
else
"- [#{title}](#{url})"
end
end)
"## #{group}\n\n#{links}"
end)
header <> sections
end
@doc """
Returns the configured LLM text sources.
"""
@spec get_sources() :: [module()]
def get_sources do
Application.get_env(:phoenix_kit, :llm_text_sources, [])
end
# Private helpers
defp get_site_name do
PhoenixKit.Settings.get_setting("site_name", "Site")
rescue
_ -> "Site"
end
defp get_site_description do
PhoenixKit.Settings.get_setting("site_description", "")
rescue
_ -> ""
end
end