Packages
html_to_markdown
2.30.0
3.9.0
3.8.3
3.8.2
3.8.1
3.8.0
3.8.0-rc.2
3.8.0-rc.1
3.7.2
3.7.1
3.7.0
3.6.21
3.6.20
3.6.19
3.6.18
3.6.17
3.6.16
3.6.15
3.6.14
3.6.13
3.6.11
3.6.10
3.6.9
3.6.8
3.6.3
3.6.1
3.6.0
3.6.0-rc.25
3.6.0-rc.23
3.6.0-rc.22
3.6.0-rc.21
3.6.0-rc.20
3.6.0-rc.17
3.6.0-rc.16
3.6.0-rc.14
3.6.0-rc.13
3.6.0-rc.12
3.6.0-rc.11
3.6.0-rc.10
3.6.0-rc.9
3.6.0-rc.7
3.6.0-rc.3
3.6.0-rc.2
3.6.0-rc.1
3.5.7
3.5.5
3.5.3
3.5.2
3.5.1
3.5.0
3.5.0-rc.1
3.4.0
3.4.0-rc.45
3.4.0-rc.44
3.4.0-rc.42
3.4.0-rc.41
3.4.0-rc.40
3.4.0-rc.25
3.4.0-rc.24
3.4.0-rc.23
3.4.0-rc.21
3.4.0-rc.20
3.4.0-rc.18
3.4.0-rc.16
3.4.0-rc.15
3.4.0-rc.14
3.4.0-rc.13
3.4.0-rc.12
3.4.0-rc.11
3.4.0-rc.9
3.4.0-rc.8
3.4.0-rc.2
3.4.0-rc.1
3.3.2
3.2.4
3.2.2
3.2.1
3.2.0
3.1.0
3.0.2
3.0.1
3.0.0
2.30.0
2.29.0
2.28.6
2.28.5
2.28.4
2.28.3
2.28.2
2.28.1
2.28.0
2.27.3
2.27.2
2.27.1
2.27.0
2.26.3
2.26.2
2.26.1
2.25.1
2.25.0
2.24.6
2.24.5
2.24.4
2.24.3
2.24.1
2.23.4
2.23.3
2.23.2
2.23.1
2.23.0
2.22.5
2.22.2
2.22.1
2.22.0
2.21.1
2.20.0
2.19.8
2.19.7
2.19.6
2.19.5
2.19.4
2.19.3
2.19.2
2.19.1
2.19.0
2.18.0
2.16.1
2.16.0
2.15.0
2.14.11
2.14.10
2.14.9
2.14.8
2.14.7
2.14.6
2.14.3
2.14.2
2.14.1
2.14.0
2.13.0
2.12.1
2.12.0
2.11.4
2.11.3
2.11.1
2.10.1
2.9.2
2.9.1
2.9.0
2.8.3
High-performance HTML to Markdown converter
Current section
Files
Jump to
Current section
Files
lib/html_to_markdown/metadata_config.ex
defmodule HtmlToMarkdown.MetadataConfig do
@moduledoc """
Configuration for metadata extraction.
This controls which metadata sections are extracted during `convert_with_metadata/3`.
"""
@default_max_structured_data_size 1_000_000
defstruct extract_document: true,
extract_headers: true,
extract_links: true,
extract_images: true,
extract_structured_data: true,
max_structured_data_size: @default_max_structured_data_size
@type t :: %__MODULE__{
extract_document: boolean(),
extract_headers: boolean(),
extract_links: boolean(),
extract_images: boolean(),
extract_structured_data: boolean(),
max_structured_data_size: pos_integer()
}
@spec new(t() | map() | keyword() | nil) :: t()
def new(nil), do: %__MODULE__{}
def new(%__MODULE__{} = cfg), do: cfg
def new(attrs) when is_list(attrs) or is_map(attrs) do
attrs =
attrs
|> Map.new()
|> Enum.reduce(%{}, fn {key, value}, acc ->
case normalize_key(key) do
nil -> acc
normalized -> Map.put(acc, normalized, value)
end
end)
struct(%__MODULE__{}, attrs)
end
defp normalize_key(key) when is_atom(key) do
if key in [
:extract_document,
:extract_headers,
:extract_links,
:extract_images,
:extract_structured_data,
:max_structured_data_size
],
do: key,
else: nil
end
defp normalize_key(key) when is_binary(key) do
key
|> String.trim()
|> String.replace("-", "_")
|> case do
"extract_document" -> :extract_document
"extract_headers" -> :extract_headers
"extract_links" -> :extract_links
"extract_images" -> :extract_images
"extract_structured_data" -> :extract_structured_data
"max_structured_data_size" -> :max_structured_data_size
_ -> nil
end
end
defp normalize_key(_), do: nil
@doc false
@spec to_map_result(t() | map() | keyword() | nil) :: {:ok, map()} | {:error, String.t()}
def to_map_result(nil), do: to_map_result(%__MODULE__{})
def to_map_result(%__MODULE__{} = cfg), do: config_to_map(cfg)
def to_map_result(attrs) when is_list(attrs) or is_map(attrs),
do: attrs |> new() |> config_to_map()
defp config_to_map(%__MODULE__{} = cfg) do
with {:ok, extract_document} <- ensure_boolean(cfg.extract_document, :extract_document),
{:ok, extract_headers} <- ensure_boolean(cfg.extract_headers, :extract_headers),
{:ok, extract_links} <- ensure_boolean(cfg.extract_links, :extract_links),
{:ok, extract_images} <- ensure_boolean(cfg.extract_images, :extract_images),
{:ok, extract_structured_data} <-
ensure_boolean(cfg.extract_structured_data, :extract_structured_data),
{:ok, max_size} <-
positive_integer(cfg.max_structured_data_size, :max_structured_data_size) do
{:ok,
%{
"extract_document" => extract_document,
"extract_headers" => extract_headers,
"extract_links" => extract_links,
"extract_images" => extract_images,
"extract_structured_data" => extract_structured_data,
"max_structured_data_size" => max_size
}}
end
end
defp ensure_boolean(value, _field) when is_boolean(value), do: {:ok, value}
defp ensure_boolean(_value, field), do: {:error, error_message(field, "must be a boolean")}
defp positive_integer(value, _field) when is_integer(value) and value > 0, do: {:ok, value}
defp positive_integer(value, field) when is_binary(value) do
value
|> String.trim()
|> Integer.parse()
|> case do
{parsed, ""} when parsed > 0 -> {:ok, parsed}
_ -> {:error, error_message(field, "must be a positive integer")}
end
end
defp positive_integer(_value, field),
do: {:error, error_message(field, "must be a positive integer")}
defp error_message(field, message) when is_atom(field), do: "#{field}: #{message}"
defp error_message(field, message), do: "#{field}: #{message}"
end