Packages
mdex
0.11.1
0.13.3
0.13.2
0.13.1
0.13.0
0.12.5
retired
0.12.4
0.12.3
0.12.2
0.12.1
0.12.0
0.11.7
0.11.6
0.11.5
0.11.4
0.11.3
0.11.2
0.11.1
0.11.0
0.10.0
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.2
0.6.1
0.6.0
0.5.0
0.4.3
0.4.2
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.0
0.1.18
0.1.17
0.1.16
0.1.15
0.1.14
0.1.13
0.1.12
0.1.11
0.1.10
0.1.9
0.1.8
0.1.7
0.1.6
0.1.5
0.1.4
0.1.3
0.1.2
0.1.1
0.1.0
Fast and extensible Markdown for Elixir
Security advisory:
This version has known vulnerabilities.
View advisories
Current section
Files
Jump to
Current section
Files
lib/mdex/fragment_parser.ex
defmodule MDEx.FragmentParser do
@moduledoc false
import NimbleParsec
@ws_chars [9, 10, 11, 12, 13, 32]
@bullet_markers ["* ", "*\t", "+ ", "+\t", "- ", "-\t"]
@task_markers ["- [ ] ", "- [ ]\t", "- [x] ", "- [x]\t", "- [X] ", "- [X]\t"]
@ordered_delims [". ", ".\t", ") ", ")\t"]
@tokens ["**", "__", "~~", "*", "_", "~"]
@fence_chars ["`", "~"]
defcombinatorp(:space_prefix, ascii_string([32], min: 0, max: 3))
defcombinatorp(:bullet_marker, choice(Enum.map(@bullet_markers, &string/1)))
defcombinatorp(:task_marker, choice(Enum.map(@task_markers, &string/1)))
defcombinatorp(
:ordered_marker,
ascii_string([?0..?9], min: 1)
|> concat(choice(Enum.map(@ordered_delims, &string/1)))
)
defcombinatorp(
:list_marker_prefix,
parsec(:space_prefix)
|> concat(choice([parsec(:task_marker), parsec(:bullet_marker), parsec(:ordered_marker)]))
)
defparsecp(:list_prefix_parser, parsec(:list_marker_prefix))
@spec complete(String.t(), keyword()) :: String.t()
def complete(fragment, options \\ []) do
prefix = options[:prefix] || ""
if list_marker_line?(fragment) do
{trailing_ws, core_with_leading} = split_trailing_ws(fragment)
{completed, flag} = complete_core(core_with_leading, prefix, trailing_ws, options)
completed <> adjust_trailing(trailing_ws, flag)
else
{leading, core, trailing} = split_ws(fragment)
leading = maybe_preserve_leading(leading)
{completed, flag} = complete_core(core, prefix, trailing, options)
leading <> completed <> adjust_trailing(trailing, flag)
end
end
defp split_trailing_ws(binary) do
{core, trailing} = take_trailing_ws(binary)
{trailing, core}
end
defp split_ws(binary) do
{leading, rest} = take_leading_ws(binary)
{core, trailing} = take_trailing_ws(rest)
{leading, core, trailing}
end
defp take_leading_ws(binary) do
size = byte_size(binary)
leading_size = leading_ws_size(binary, size, 0)
{
binary_part(binary, 0, leading_size),
binary_part(binary, leading_size, size - leading_size)
}
end
defp take_trailing_ws(binary) do
size = byte_size(binary)
trailing_size = trailing_ws_size(binary, size - 1, 0)
core_size = size - trailing_size
{
binary_part(binary, 0, core_size),
binary_part(binary, core_size, trailing_size)
}
end
defp adjust_trailing(trailing, {:consume_trailing, consumed}) when consumed != "" do
if String.starts_with?(trailing, consumed) do
drop_prefix(trailing, consumed)
else
trailing
end
end
defp adjust_trailing(trailing, _flag), do: trailing
defp complete_core("", _prefix, _trailing, _options), do: {"", :none}
defp complete_core(core, prefix, trailing, options) do
with nil <- maybe_complete_fence(core, trailing, options),
nil <- maybe_complete_table(core, trailing),
nil <- maybe_complete_math(core, trailing) do
line = last_line(core)
opening_append = opening_completion_append(core)
cond do
skip_completion?(core, line) ->
{core, :none}
unmatched_backtick?(core) ->
if prefix_has_open_backtick?(prefix) and String.ends_with?(core, "`") do
{prefix <> core, :none}
else
{core <> "`", :backtick_appended}
end
opening_append != "" ->
{core <> opening_append, :none}
closer = closing_token(core) ->
if prefix != "" and prefix == closer do
{prefix <> core, :none}
else
{core, :none}
end
list_marker_line?(line) ->
{complete_list_line(core, line), :none}
true ->
case incomplete_link_completion(core) do
nil ->
case unmatched_suffix(line) do
suffix when suffix != "" -> {core <> suffix, :none}
_ -> {core, :none}
end
completion ->
{completion, :none}
end
end
end
end
defp maybe_complete_fence(core, trailing, _options) do
case unclosed_fence_info(core) do
nil ->
nil
%{indent: indent, char: char, run: run} = info ->
case fence_partial_closing_gap(core, info) do
{:append, append} ->
{core <> append, :none}
:none ->
# reuse a trailing newline when available so the appended delimiter matches expectations
{consumed, _remaining?} = consume_trailing_for_fence(trailing)
closing_line = indent <> String.duplicate(char, run)
leading_segment =
cond do
consumed != "" -> consumed
String.ends_with?(core, "\n") -> ""
true -> "\n"
end
append = leading_segment <> closing_line
flag = if consumed == "", do: :none, else: {:consume_trailing, consumed}
{core <> append, flag}
end
end
end
defp fence_partial_closing_gap(core, %{indent: indent, char: char, run: run}) do
case parse_fence_line(last_line(core)) do
%{indent: ^indent, char: ^char, run: partial_run, rest: rest}
when partial_run > 0 and partial_run < run ->
if only_spaces?(rest) do
missing = run - partial_run
{:append, String.duplicate(char, missing)}
else
:none
end
_ ->
:none
end
end
defp unclosed_fence_info(core) do
core
|> String.split("\n", trim: false)
|> find_unclosed_fence(0, nil)
|> normalize_fence_info()
end
defp find_unclosed_fence([], _index, open_fence), do: open_fence
defp find_unclosed_fence([line | rest], index, open_fence) do
case parse_fence_line(line) do
%{run: run} = info when run >= 3 ->
updated_info = Map.merge(info, %{line_index: index, has_body: false})
cond do
open_fence == nil ->
find_unclosed_fence(rest, index + 1, updated_info)
closes_open_fence?(updated_info, open_fence) ->
find_unclosed_fence(rest, index + 1, nil)
true ->
find_unclosed_fence(rest, index + 1, updated_info)
end
_ ->
open_fence = maybe_mark_body(open_fence, index)
find_unclosed_fence(rest, index + 1, open_fence)
end
end
defp maybe_mark_body(nil, _index), do: nil
defp maybe_mark_body(%{line_index: line_index} = info, index) when index > line_index do
Map.put(info, :has_body, true)
end
defp maybe_mark_body(info, _index), do: info
defp normalize_fence_info(nil), do: nil
defp normalize_fence_info(%{has_body: false}), do: nil
defp normalize_fence_info(info), do: Map.drop(info, [:has_body])
defp closes_open_fence?(%{indent: indent, char: char, run: run, rest: rest}, %{indent: indent, char: char, run: open_run}) do
run >= open_run and only_spaces?(rest)
end
defp parse_fence_line(line) do
{indent, rest} = take_fence_indent(line)
with <<char_code, _::binary>> <- rest,
char = <<char_code>>,
true <- char in @fence_chars do
run = count_fence_run(rest, char_code, 0)
remainder = binary_part(rest, run, byte_size(rest) - run)
%{indent: indent, char: char, run: run, rest: remainder}
else
_ -> nil
end
end
defp take_fence_indent(line), do: take_fence_indent(line, 0)
defp take_fence_indent(<<" "::utf8, rest::binary>>, count) when count < 3 do
take_fence_indent(rest, count + 1)
end
defp take_fence_indent(rest, count) do
{String.duplicate(" ", count), rest}
end
defp count_fence_run(<<char_code, rest::binary>>, char_code, acc) do
count_fence_run(rest, char_code, acc + 1)
end
defp count_fence_run(_rest, _char_code, acc), do: acc
defp only_spaces?(""), do: true
defp only_spaces?(<<char, rest::binary>>) when char in [32, 9], do: only_spaces?(rest)
defp only_spaces?(_), do: false
defp consume_trailing_for_fence(trailing) do
if String.starts_with?(trailing, "\n") do
{"\n", drop_prefix(trailing, "\n")}
else
{"", trailing}
end
end
defp drop_prefix(binary, prefix) do
prefix_size = byte_size(prefix)
binary_part(binary, prefix_size, byte_size(binary) - prefix_size)
end
@doc false
def merge_stream_buffer(existing_markdown, buffer, last_node) do
existing_markdown
|> maybe_strip_trailing_fence(last_node)
|> ensure_trailing_newline()
|> append_buffer(buffer)
end
defp maybe_strip_trailing_fence(markdown, %{__struct__: MDEx.CodeBlock, fenced: true} = block) do
closing_line = closing_fence_line(block)
trim_trailing_line(markdown, closing_line)
end
defp maybe_strip_trailing_fence(markdown, _), do: markdown
defp ensure_trailing_newline(<<>>), do: ""
defp ensure_trailing_newline(binary) do
case :binary.last(binary) do
?\n -> binary
_ -> binary <> "\n"
end
end
defp append_buffer(existing, buffer) do
IO.iodata_to_binary([existing | Enum.reverse(buffer)])
end
defp closing_fence_line(%{fence_char: char, fence_length: length, fence_offset: offset}) do
indent = if offset > 0, do: String.duplicate(" ", offset), else: ""
indent <> String.duplicate(char, length)
end
defp trim_trailing_line(markdown, closing_line) do
newline_closing = "\n" <> closing_line
cond do
String.ends_with?(markdown, newline_closing <> "\n") ->
drop_suffix(markdown, byte_size(newline_closing <> "\n"))
String.ends_with?(markdown, newline_closing) ->
drop_suffix(markdown, byte_size(newline_closing))
true ->
markdown
end
end
defp drop_suffix(markdown, size_to_drop) do
length = byte_size(markdown)
binary_part(markdown, 0, length - size_to_drop)
end
defp whitespace_char?(char), do: char in @ws_chars
defp leading_ws_size(_binary, size, size), do: size
defp leading_ws_size(binary, size, index) do
char = :binary.at(binary, index)
if whitespace_char?(char) do
leading_ws_size(binary, size, index + 1)
else
index
end
end
defp trailing_ws_size(_binary, index, count) when index < 0, do: count
defp trailing_ws_size(binary, index, count) do
char = :binary.at(binary, index)
if whitespace_char?(char) do
trailing_ws_size(binary, index - 1, count + 1)
else
count
end
end
defp maybe_preserve_leading(""), do: ""
defp maybe_preserve_leading(leading) do
if preserve_leading?(leading) do
leading
else
""
end
end
defp preserve_leading?(leading) do
contains_line_break?(leading) or code_block_indented?(leading)
end
defp contains_line_break?(leading) do
:binary.match(leading, <<10>>) != :nomatch or :binary.match(leading, <<13>>) != :nomatch
end
defp code_block_indented?(leading) do
:binary.match(leading, <<9>>) != :nomatch or final_space_run(leading) >= 4
end
defp final_space_run(binary), do: final_space_run(binary, 0)
defp final_space_run(<<>>, count), do: count
defp final_space_run(<<10, rest::binary>>, _count), do: final_space_run(rest, 0)
defp final_space_run(<<13, rest::binary>>, _count), do: final_space_run(rest, 0)
defp final_space_run(<<32, rest::binary>>, count), do: final_space_run(rest, count + 1)
defp final_space_run(<<_char, rest::binary>>, count), do: final_space_run(rest, count)
defp maybe_complete_table(core, trailing) do
if String.starts_with?(trailing, "\n") do
line = last_line(core)
if table_header_line?(line) do
separator = generate_table_separator(line)
{core <> "\n" <> separator, {:consume_trailing, "\n"}}
else
nil
end
else
nil
end
end
defp table_header_line?(line) do
trimmed = String.trim(line)
String.starts_with?(trimmed, "|") and String.ends_with?(trimmed, "|")
end
defp generate_table_separator(line) do
pipe_count = count_char(line, ?|)
cond do
pipe_count > 1 ->
columns = pipe_count - 1
columns
|> build_table_separator(["|"])
|> IO.iodata_to_binary()
true ->
""
end
end
defp build_table_separator(0, acc), do: acc
defp build_table_separator(columns, acc) do
build_table_separator(columns - 1, [acc, " - |"])
end
defp maybe_complete_math(core, trailing) do
cond do
has_unclosed_delimiter?(core, "$$") ->
if String.starts_with?(trailing, "\n") do
{core <> "\n$$", {:consume_trailing, "\n"}}
else
{core <> "$$", :none}
end
has_unclosed_delimiter?(core, "$") ->
{core <> "$", :none}
true ->
nil
end
end
defp has_unclosed_delimiter?(text, delimiter) do
count = count_occurrences(text, delimiter)
rem(count, 2) == 1
end
defp complete_list_line(core, line) do
case extract_list_content(line) do
{prefix, content} when content != "" ->
case unmatched_suffix(content) do
suffix when suffix != "" ->
new_line = prefix <> content <> suffix
replace_last_line(core, new_line)
"" ->
case incomplete_link_completion(content) do
nil -> core
completion -> replace_last_line(core, prefix <> completion)
end
end
_ ->
core
end
end
defp skip_completion?(core, line) do
fence_candidate?(core) or fence_line_start?(line)
end
defp unmatched_suffix(text) do
Enum.find_value([{"*", "**"}, {"_", "__"}, {"~", "~~"}], "", fn {single, double} ->
case unmatched_emph_suffix(text, single, double) do
suffix when suffix != "" -> suffix
_ -> nil
end
end)
end
defp opening_completion_append(core) do
with token when token != nil <- opening_token(core),
append when append != "" <- missing_trailing_for(core, token),
true <- rem(count_occurrences(core, token), 2) == 1 do
append
else
_ -> ""
end
end
defp opening_token(bin) do
Enum.find(@tokens, &String.starts_with?(bin, &1))
end
defp closing_token(bin) do
Enum.find(@tokens, &String.ends_with?(bin, &1))
end
defp missing_trailing_for(bin, token) do
with char when char != nil <- String.first(token),
needed = div(String.length(token), String.length(char)),
present = trailing_run_len(bin, char),
true <- present < needed do
String.duplicate(char, needed - present)
else
_ -> ""
end
end
defp trailing_run_len(bin, char) do
char_len = String.length(char)
if char_len == 0 do
0
else
consumed = String.length(bin) - String.length(String.trim_trailing(bin, char))
div(consumed, char_len)
end
end
defp unmatched_backtick?(bin), do: rem(count_char(bin, ?`), 2) == 1
defp prefix_has_open_backtick?(""), do: false
defp prefix_has_open_backtick?(prefix), do: unmatched_backtick?(prefix)
defp unmatched_emph_suffix(core, single, double) do
single_code = single |> String.to_charlist() |> List.first()
total = count_emphasis_char(core, single_code)
double_count = count_occurrences(core, double)
unmatched_double = rem(double_count, 2)
unmatched_single = rem(max(total - double_count * 2, 0), 2)
cond do
unmatched_double == 1 and unmatched_single == 1 ->
if String.ends_with?(core, single) and not String.ends_with?(core, double) do
single
else
double <> single
end
unmatched_double == 1 ->
double
unmatched_single == 1 ->
single
true ->
""
end
end
defp count_emphasis_char(bin, ?_) do
count_emphasis_underscore(bin)
end
defp count_emphasis_char(bin, char) do
count_char(bin, char)
end
defp count_char(bin, char) when is_integer(char) do
bin |> :binary.matches(<<char>>) |> count_matches()
end
defp count_occurrences(bin, token) do
bin |> :binary.matches(token) |> count_matches()
end
defp count_matches(:nomatch), do: 0
defp count_matches(matches), do: length(matches)
defp count_emphasis_underscore(bin) do
count_emphasis_underscore(bin, :boundary, 0)
end
defp count_emphasis_underscore(<<>>, _prev_type, acc), do: acc
defp count_emphasis_underscore(<<char::utf8, rest::binary>>, prev_type, acc) do
cond do
char == ?_ ->
next_type = next_char_type(rest)
eligible? = not (prev_type == :word and next_type == :word)
new_acc = if eligible?, do: acc + 1, else: acc
count_emphasis_underscore(rest, :underscore, new_acc)
true ->
count_emphasis_underscore(rest, char_type(char), acc)
end
end
defp next_char_type(<<>>), do: :boundary
defp next_char_type(<<char::utf8, _rest::binary>>), do: char_type(char)
defp char_type(char) when char in ?0..?9, do: :word
defp char_type(char) when char in ?A..?Z, do: :word
defp char_type(char) when char in ?a..?z, do: :word
defp char_type(_char), do: :boundary
defp incomplete_link_completion(core) do
cond do
incomplete_link_brackets?(core) ->
ensure_placeholder_prefix(core, "](mdex:incomplete-link)")
incomplete_link_destination?(core) ->
ensure_placeholder_prefix(core, "(mdex:incomplete-link)")
true ->
nil
end
end
defp ensure_placeholder_prefix(core, placeholder) do
if String.ends_with?(core, placeholder), do: core, else: core <> placeholder
end
defp incomplete_link_brackets?(core) do
count_occurrences(core, "[") > count_occurrences(core, "]")
end
defp incomplete_link_destination?(core) do
case trailing_link_label(core) do
nil -> false
_span -> true
end
end
defp fence_candidate?(""), do: false
defp fence_candidate?(bin) do
not String.contains?(bin, "\n") and Enum.any?(@fence_chars, &(leading_run_len(bin, &1) >= 3))
end
defp fence_line_start?(line) do
Enum.any?(@fence_chars, &(leading_run_len(line, &1) >= 3))
end
defp leading_run_len("", _char), do: 0
defp leading_run_len(bin, char) do
char_len = String.length(char)
if char_len == 0 do
0
else
consumed = String.length(bin) - String.length(String.trim_leading(bin, char))
div(consumed, char_len)
end
end
defp last_line(""), do: ""
defp last_line(bin) do
case find_last_newline(bin, byte_size(bin) - 1) do
nil ->
bin
index ->
start = index + 1
binary_part(bin, start, byte_size(bin) - start)
end
end
defp replace_last_line("", new_line), do: new_line
defp replace_last_line(bin, new_line) do
case find_last_newline(bin, byte_size(bin) - 1) do
nil ->
new_line
index ->
binary_part(bin, 0, index + 1) <> new_line
end
end
defp find_last_newline(_binary, index) when index < 0, do: nil
defp find_last_newline(binary, index) do
case :binary.at(binary, index) do
?\n -> index
_ -> find_last_newline(binary, index - 1)
end
end
defp list_marker_line?(line) do
match?({:ok, _, _, _, _, _}, list_prefix_parser(line))
end
defp extract_list_content(line) do
case list_prefix_parser(line) do
{:ok, parts, rest, _, _, _} -> {Enum.join(parts), rest}
_ -> {"", line}
end
end
defp trailing_link_label(<<>>), do: nil
defp trailing_link_label(core) do
size = byte_size(core)
if size == 0 do
nil
else
last_index = size - 1
case :binary.at(core, last_index) do
?] ->
with start when not is_nil(start) <- find_label_start(core, last_index - 1) do
{start, last_index - start + 1}
else
_ -> nil
end
_ ->
nil
end
end
end
defp find_label_start(_binary, index) when index < 0, do: nil
defp find_label_start(binary, index) do
case :binary.at(binary, index) do
?[ -> index
?] -> nil
_ -> find_label_start(binary, index - 1)
end
end
end