Current section

Files

Jump to
eyeon lib eyeon parser.ex
Raw

lib/eyeon/parser.ex

defmodule Eyeon.Parser do
@moduledoc false
@spec parse(binary()) :: {:ok, list()} | {:error, any()}
def parse(str) do
utf8 = ensure_utf8(str)
case utf8 |> to_charlist() |> :ion_lexer.string() do
{:ok, [], _} ->
{:ok, []}
{:ok, tokens, _} ->
with :ok <- Eyeon.Adjacency.validate(tokens) do
tokens |> Enum.reject(&whitespace_or_comment?/1) |> parse_filtered()
end
{:error, {line, :ion_lexer, {:user, message}}, _} ->
{:error, "lexer error on line #{line}: #{message}"}
{:error, {line, :ion_lexer, error}, _} ->
{:error, "lexer error on line #{line}: #{inspect(error)}"}
end
end
# BOM-based detection
# UTF-32 BE BOM
defp ensure_utf8(<<0x00, 0x00, 0xFE, 0xFF, rest::binary>>), do: transcode_utf32_be(rest)
# UTF-32 LE BOM (must come before UTF-16 LE since FF FE is a prefix)
defp ensure_utf8(<<0xFF, 0xFE, 0x00, 0x00, rest::binary>>), do: transcode_utf32_le(rest)
# UTF-16 BE BOM
defp ensure_utf8(<<0xFE, 0xFF, rest::binary>>), do: transcode_utf16_be(rest)
# UTF-16 LE BOM
defp ensure_utf8(<<0xFF, 0xFE, rest::binary>>), do: transcode_utf16_le(rest)
# UTF-8 BOM
defp ensure_utf8(<<0xEF, 0xBB, 0xBF, rest::binary>>), do: rest
# Heuristic detection for BOM-less files
# UTF-32 BE: 00 00 00 xx
defp ensure_utf8(<<0x00, 0x00, 0x00, c, _::binary>> = bin) when c != 0x00,
do: transcode_utf32_be(bin)
# UTF-32 LE: xx 00 00 00
defp ensure_utf8(<<c, 0x00, 0x00, 0x00, _::binary>> = bin) when c != 0x00,
do: transcode_utf32_le(bin)
# UTF-16 BE: 00 xx (where xx is a plausible ASCII char)
defp ensure_utf8(<<0x00, c, _::binary>> = bin) when c != 0x00, do: transcode_utf16_be(bin)
# UTF-16 LE: xx 00 (where xx is a plausible ASCII char)
defp ensure_utf8(<<c, 0x00, _::binary>> = bin) when c != 0x00, do: transcode_utf16_le(bin)
# Already UTF-8 (or empty)
defp ensure_utf8(bin), do: bin
defp transcode_utf16_be(bin), do: do_transcode_utf16(bin, :big)
defp transcode_utf16_le(bin), do: do_transcode_utf16(bin, :little)
defp do_transcode_utf16(bin, endian) do
bin
|> decode_utf16_codepoints(endian, [])
|> IO.iodata_to_binary()
end
defp decode_utf16_codepoints(<<>>, _endian, acc), do: Enum.reverse(acc)
# Surrogate pair (U+10000 and above)
defp decode_utf16_codepoints(
<<hi::size(16)-integer, lo::size(16)-integer, rest::binary>>,
:big,
acc
)
when hi >= 0xD800 and hi <= 0xDBFF and lo >= 0xDC00 and lo <= 0xDFFF do
codepoint = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000
decode_utf16_codepoints(rest, :big, [<<codepoint::utf8>> | acc])
end
defp decode_utf16_codepoints(
<<hi::size(16)-little-integer, lo::size(16)-little-integer, rest::binary>>,
:little,
acc
)
when hi >= 0xD800 and hi <= 0xDBFF and lo >= 0xDC00 and lo <= 0xDFFF do
codepoint = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000
decode_utf16_codepoints(rest, :little, [<<codepoint::utf8>> | acc])
end
# Basic Multilingual Plane
defp decode_utf16_codepoints(<<cp::size(16)-integer, rest::binary>>, :big, acc) do
decode_utf16_codepoints(rest, :big, [<<cp::utf8>> | acc])
end
defp decode_utf16_codepoints(<<cp::size(16)-little-integer, rest::binary>>, :little, acc) do
decode_utf16_codepoints(rest, :little, [<<cp::utf8>> | acc])
end
defp transcode_utf32_be(bin), do: do_transcode_utf32(bin, :big)
defp transcode_utf32_le(bin), do: do_transcode_utf32(bin, :little)
defp do_transcode_utf32(bin, endian) do
bin
|> decode_utf32_codepoints(endian, [])
|> IO.iodata_to_binary()
end
defp decode_utf32_codepoints(<<>>, _endian, acc), do: Enum.reverse(acc)
defp decode_utf32_codepoints(<<cp::size(32)-integer, rest::binary>>, :big, acc) do
decode_utf32_codepoints(rest, :big, [<<cp::utf8>> | acc])
end
defp decode_utf32_codepoints(<<cp::size(32)-little-integer, rest::binary>>, :little, acc) do
decode_utf32_codepoints(rest, :little, [<<cp::utf8>> | acc])
end
defp parse_filtered([]), do: {:ok, []}
defp parse_filtered(tokens) do
with {:ok, ast} <- :ion_parser.parse(tokens) do
Eyeon.MacroExpander.expand(ast)
end
end
defp whitespace_or_comment?({:whitespace, _, _}), do: true
defp whitespace_or_comment?({:comment, _, _}), do: true
defp whitespace_or_comment?(_), do: false
end