Current section
Files
Jump to
Current section
Files
lib/eyeon/text_parser/string.ex
defmodule Eyeon.TextParser.String do
@moduledoc false
# Parses Ion short strings, long strings, quoted symbols, and identifier symbols.
# Each function takes (binary, pos, opts) and returns {value, rest, pos} or {:error, reason}.
@doc """
Parse a short string (already consumed the opening `"`).
Returns {string, rest, pos}.
"""
@spec parse_short_string(binary(), non_neg_integer(), map()) ::
{String.t(), binary(), non_neg_integer()} | {:error, String.t()}
def parse_short_string(data, pos, _opts) do
scan_short_string(data, pos, [])
end
defp scan_short_string(<<>>, _pos, _acc) do
{:error, "unexpected end of input in string"}
end
defp scan_short_string(<<?\", rest::binary>>, pos, acc) do
str = acc |> Enum.reverse() |> IO.iodata_to_binary()
{str, rest, pos + 1}
end
# Line continuation: backslash followed by newline
defp scan_short_string(<<?\\, ?\r, ?\n, rest::binary>>, pos, acc) do
scan_short_string(rest, pos + 3, acc)
end
defp scan_short_string(<<?\\, ?\n, rest::binary>>, pos, acc) do
scan_short_string(rest, pos + 2, acc)
end
defp scan_short_string(<<?\\, ?\r, rest::binary>>, pos, acc) do
scan_short_string(rest, pos + 2, acc)
end
# Escape sequences
defp scan_short_string(<<?\\, rest::binary>>, pos, acc) do
case parse_string_escape(rest, pos + 1) do
{:error, _} = err -> err
{bytes, rest2, pos2} -> scan_short_string(rest2, pos2, [bytes | acc])
end
end
# Bare newlines are not allowed in short strings
defp scan_short_string(<<?\n, _rest::binary>>, _pos, _acc) do
{:error, "unescaped newline in short string"}
end
defp scan_short_string(<<?\r, _rest::binary>>, _pos, _acc) do
{:error, "unescaped newline in short string"}
end
# Reject raw control characters (0x00-0x1F except TAB, VT, FF which are allowed)
defp scan_short_string(<<byte, _rest::binary>>, _pos, _acc)
when byte < 0x20 and byte != ?\t and byte != 0x0B and byte != 0x0C do
{:error, "raw control character in string"}
end
defp scan_short_string(<<byte, rest::binary>>, pos, acc) do
scan_short_string(rest, pos + 1, [byte | acc])
end
@doc """
Parse a long string (already consumed the opening `'''`).
Returns {string, rest, pos}.
After closing `'''`, checks for concatenated long strings.
"""
@spec parse_long_string(binary(), non_neg_integer(), map()) ::
{String.t(), binary(), non_neg_integer()} | {:error, String.t()}
def parse_long_string(data, pos, opts) do
parse_long_string_parts(data, pos, opts, [])
end
defp parse_long_string_parts(data, pos, opts, parts) do
case scan_long_string(data, pos, []) do
{:error, _} = err ->
err
{part, rest, pos2} ->
all_parts = [part | parts]
# Skip whitespace and check for another '''
{rest2, pos3} = skip_ws_only(rest, pos2)
case rest2 do
<<"'''", rest3::binary>> ->
parse_long_string_parts(rest3, pos3 + 3, opts, all_parts)
_ ->
str = all_parts |> Enum.reverse() |> IO.iodata_to_binary()
{str, rest2, pos3}
end
end
end
# Skip whitespace and comments for long string concatenation check
defp skip_ws_only(<<byte, rest::binary>>, pos) when byte in [?\s, ?\t, ?\n, ?\r, ?\v, ?\f] do
skip_ws_only(rest, pos + 1)
end
# Line comment
defp skip_ws_only(<<?/, ?/, rest::binary>>, pos) do
skip_ws_line_comment(rest, pos + 2)
end
# Block comment
defp skip_ws_only(<<?/, ?*, rest::binary>>, pos) do
skip_ws_block_comment(rest, pos + 2)
end
defp skip_ws_only(data, pos), do: {data, pos}
defp skip_ws_line_comment(<<?\n, rest::binary>>, pos), do: skip_ws_only(rest, pos + 1)
defp skip_ws_line_comment(<<?\r, ?\n, rest::binary>>, pos), do: skip_ws_only(rest, pos + 2)
defp skip_ws_line_comment(<<?\r, rest::binary>>, pos), do: skip_ws_only(rest, pos + 1)
defp skip_ws_line_comment(<<>>, pos), do: {<<>>, pos}
defp skip_ws_line_comment(<<_, rest::binary>>, pos), do: skip_ws_line_comment(rest, pos + 1)
defp skip_ws_block_comment(<<?*, ?/, rest::binary>>, pos), do: skip_ws_only(rest, pos + 2)
defp skip_ws_block_comment(<<>>, pos), do: {<<>>, pos}
defp skip_ws_block_comment(<<_, rest::binary>>, pos), do: skip_ws_block_comment(rest, pos + 1)
defp scan_long_string(<<>>, _pos, _acc) do
{:error, "unexpected end of input in long string"}
end
defp scan_long_string(<<"'''", rest::binary>>, pos, acc) do
str = acc |> Enum.reverse() |> IO.iodata_to_binary()
{str, rest, pos + 3}
end
# Escape sequences
defp scan_long_string(<<?\\, ?\r, ?\n, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 3, acc)
end
defp scan_long_string(<<?\\, ?\n, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 2, acc)
end
defp scan_long_string(<<?\\, ?\r, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 2, acc)
end
defp scan_long_string(<<?\\, rest::binary>>, pos, acc) do
case parse_string_escape(rest, pos + 1) do
{:error, _} = err -> err
{bytes, rest2, pos2} -> scan_long_string(rest2, pos2, [bytes | acc])
end
end
# Newlines are allowed in long strings - normalize \r\n and \r to \n
defp scan_long_string(<<?\r, ?\n, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 2, [?\n | acc])
end
defp scan_long_string(<<?\r, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 1, [?\n | acc])
end
# Raw newlines are allowed in long strings
defp scan_long_string(<<?\n, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 1, [?\n | acc])
end
# Reject raw control characters (0x00-0x1F except TAB, LF, CR, VT, FF)
defp scan_long_string(<<byte, _rest::binary>>, _pos, _acc)
when byte < 0x20 and byte != ?\t and byte != 0x0B and byte != 0x0C do
{:error, "raw control character in long string"}
end
defp scan_long_string(<<byte, rest::binary>>, pos, acc) do
scan_long_string(rest, pos + 1, [byte | acc])
end
@doc """
Parse a quoted symbol (already consumed the opening `'`).
Returns {{:symbol, name}, rest, pos}.
"""
@spec parse_quoted_symbol(binary(), non_neg_integer(), map()) ::
{{:symbol, String.t()}, binary(), non_neg_integer()} | {:error, String.t()}
def parse_quoted_symbol(data, pos, _opts) do
case scan_quoted_symbol(data, pos, []) do
{:error, _} = err -> err
{name, rest, pos2} -> {{:symbol, name}, rest, pos2}
end
end
defp scan_quoted_symbol(<<>>, _pos, _acc) do
{:error, "unexpected end of input in quoted symbol"}
end
defp scan_quoted_symbol(<<?\', rest::binary>>, pos, acc) do
str = acc |> Enum.reverse() |> IO.iodata_to_binary()
{str, rest, pos + 1}
end
defp scan_quoted_symbol(<<?\n, _rest::binary>>, _pos, _acc) do
{:error, "unescaped newline in quoted symbol"}
end
defp scan_quoted_symbol(<<?\r, _rest::binary>>, _pos, _acc) do
{:error, "unescaped newline in quoted symbol"}
end
# Line continuation in quoted symbols: backslash followed by newline
defp scan_quoted_symbol(<<?\\, ?\r, ?\n, rest::binary>>, pos, acc) do
scan_quoted_symbol(rest, pos + 3, acc)
end
defp scan_quoted_symbol(<<?\\, ?\n, rest::binary>>, pos, acc) do
scan_quoted_symbol(rest, pos + 2, acc)
end
defp scan_quoted_symbol(<<?\\, ?\r, rest::binary>>, pos, acc) do
scan_quoted_symbol(rest, pos + 2, acc)
end
defp scan_quoted_symbol(<<?\\, rest::binary>>, pos, acc) do
case parse_string_escape(rest, pos + 1) do
{:error, _} = err -> err
{bytes, rest2, pos2} -> scan_quoted_symbol(rest2, pos2, [bytes | acc])
end
end
defp scan_quoted_symbol(<<byte, rest::binary>>, pos, acc) do
scan_quoted_symbol(rest, pos + 1, [byte | acc])
end
@doc """
Parse an identifier symbol or keyword.
The first character has already been verified as [$_a-zA-Z].
`first` is the first byte, data starts after it.
Returns {identifier_string, rest, pos}.
"""
@spec parse_identifier(byte(), binary(), non_neg_integer()) ::
{String.t(), binary(), non_neg_integer()}
def parse_identifier(first, data, pos) do
scan_identifier(data, pos, [first])
end
defp scan_identifier(<<byte, rest::binary>>, pos, acc)
when byte in ?a..?z or byte in ?A..?Z or byte in ?0..?9 or byte == ?_ or byte == ?$ do
scan_identifier(rest, pos + 1, [byte | acc])
end
defp scan_identifier(data, pos, acc) do
str = acc |> Enum.reverse() |> IO.iodata_to_binary()
{str, data, pos}
end
# ---- Clob string parsers ----
# Clobs only allow ASCII bytes (0x20..0x7E) in the content,
# and only allow \x hex escapes (not \u or \U unicode escapes).
@doc """
Parse a short string for clob content (already consumed the opening `"`).
Validates that all bytes are ASCII and rejects unicode escapes.
"""
@spec parse_clob_short_string(binary(), non_neg_integer(), map()) ::
{String.t(), binary(), non_neg_integer()} | {:error, String.t()}
def parse_clob_short_string(data, pos, _opts) do
scan_clob_short_string(data, pos, [])
end
defp scan_clob_short_string(<<>>, _pos, _acc) do
{:error, "unexpected end of input in clob string"}
end
defp scan_clob_short_string(<<?\", rest::binary>>, pos, acc) do
str = acc |> Enum.reverse() |> IO.iodata_to_binary()
# Validate all bytes are in the ASCII printable + allowed escape range
validate_clob_content(str, rest, pos + 1)
end
# Line continuation
defp scan_clob_short_string(<<?\\, ?\r, ?\n, rest::binary>>, pos, acc) do
scan_clob_short_string(rest, pos + 3, acc)
end
defp scan_clob_short_string(<<?\\, ?\n, rest::binary>>, pos, acc) do
scan_clob_short_string(rest, pos + 2, acc)
end
defp scan_clob_short_string(<<?\\, ?\r, rest::binary>>, pos, acc) do
scan_clob_short_string(rest, pos + 2, acc)
end
# Reject unicode escapes in clobs
defp scan_clob_short_string(<<?\\, ?u, _::binary>>, _pos, _acc) do
{:error, "unicode escape \\u not allowed in clob"}
end
defp scan_clob_short_string(<<?\\, ?U, _::binary>>, _pos, _acc) do
{:error, "unicode escape \\U not allowed in clob"}
end
defp scan_clob_short_string(<<?\\, rest::binary>>, pos, acc) do
case parse_escape(rest, pos + 1) do
{:error, _} = err -> err
{bytes, rest2, pos2} -> scan_clob_short_string(rest2, pos2, [bytes | acc])
end
end
defp scan_clob_short_string(<<?\n, _rest::binary>>, _pos, _acc) do
{:error, "unescaped newline in clob short string"}
end
defp scan_clob_short_string(<<?\r, _rest::binary>>, _pos, _acc) do
{:error, "unescaped newline in clob short string"}
end
# Reject raw control characters in clobs (0x00-0x1F except tab, VT, FF)
defp scan_clob_short_string(<<byte, _rest::binary>>, _pos, _acc)
when byte < 0x20 and byte != ?\t and byte != 0x0B and byte != 0x0C do
{:error, "raw control character in clob"}
end
defp scan_clob_short_string(<<byte, _rest::binary>>, _pos, _acc)
when byte > 0x7F do
{:error, "non-ASCII byte in clob"}
end
defp scan_clob_short_string(<<byte, rest::binary>>, pos, acc) do
scan_clob_short_string(rest, pos + 1, [byte | acc])
end
defp validate_clob_content(str, rest, pos) do
{str, rest, pos}
end
@doc """
Parse a long string for clob content (already consumed the opening `'''`).
Validates that all bytes are ASCII and rejects unicode escapes.
"""
@spec parse_clob_long_string(binary(), non_neg_integer(), map()) ::
{String.t(), binary(), non_neg_integer()} | {:error, String.t()}
def parse_clob_long_string(data, pos, _opts) do
scan_clob_long_string(data, pos, [])
end
defp scan_clob_long_string(<<>>, _pos, _acc) do
{:error, "unexpected end of input in clob long string"}
end
defp scan_clob_long_string(<<"'''", rest::binary>>, pos, acc) do
str = acc |> Enum.reverse() |> IO.iodata_to_binary()
{str, rest, pos + 3}
end
# Escape sequences
defp scan_clob_long_string(<<?\\, ?\r, ?\n, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 3, acc)
end
defp scan_clob_long_string(<<?\\, ?\n, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 2, acc)
end
defp scan_clob_long_string(<<?\\, ?\r, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 2, acc)
end
# Reject unicode escapes in clobs
defp scan_clob_long_string(<<?\\, ?u, _::binary>>, _pos, _acc) do
{:error, "unicode escape \\u not allowed in clob"}
end
defp scan_clob_long_string(<<?\\, ?U, _::binary>>, _pos, _acc) do
{:error, "unicode escape \\U not allowed in clob"}
end
defp scan_clob_long_string(<<?\\, rest::binary>>, pos, acc) do
case parse_escape(rest, pos + 1) do
{:error, _} = err -> err
{bytes, rest2, pos2} -> scan_clob_long_string(rest2, pos2, [bytes | acc])
end
end
# Newlines are allowed in long clob strings - normalize to \n
defp scan_clob_long_string(<<?\r, ?\n, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 2, [?\n | acc])
end
defp scan_clob_long_string(<<?\r, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 1, [?\n | acc])
end
# Raw newlines are allowed in long clob strings
defp scan_clob_long_string(<<?\n, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 1, [?\n | acc])
end
# Reject raw control characters in long clob strings (except TAB, LF, CR, VT, FF)
defp scan_clob_long_string(<<byte, _rest::binary>>, _pos, _acc)
when byte < 0x20 and byte != ?\t and byte != 0x0B and byte != 0x0C do
{:error, "raw control character in clob"}
end
defp scan_clob_long_string(<<byte, _rest::binary>>, _pos, _acc)
when byte > 0x7F do
{:error, "non-ASCII byte in clob"}
end
defp scan_clob_long_string(<<byte, rest::binary>>, pos, acc) do
scan_clob_long_string(rest, pos + 1, [byte | acc])
end
# ---- Escape sequence parser ----
# Takes the data AFTER the backslash and returns {bytes, rest, pos}
# String/symbol variant: \xHH produces UTF-8 encoded codepoint.
# Clobs use parse_escape/2 directly which produces raw bytes.
defp parse_string_escape(<<?x, h1, h2, rest::binary>>, pos)
when (h1 in ?0..?9 or h1 in ?a..?f or h1 in ?A..?F) and
(h2 in ?0..?9 or h2 in ?a..?f or h2 in ?A..?F) do
codepoint = hex_val(h1) * 16 + hex_val(h2)
{<<codepoint::utf8>>, rest, pos + 3}
end
defp parse_string_escape(data, pos), do: parse_escape(data, pos)
@spec parse_escape(binary(), non_neg_integer()) ::
{iodata(), binary(), non_neg_integer()} | {:error, String.t()}
def parse_escape(<<>>, _pos), do: {:error, "unexpected end of input after backslash"}
def parse_escape(<<?a, rest::binary>>, pos), do: {"\a", rest, pos + 1}
def parse_escape(<<?b, rest::binary>>, pos), do: {"\b", rest, pos + 1}
def parse_escape(<<?t, rest::binary>>, pos), do: {"\t", rest, pos + 1}
def parse_escape(<<?n, rest::binary>>, pos), do: {"\n", rest, pos + 1}
def parse_escape(<<?f, rest::binary>>, pos), do: {"\f", rest, pos + 1}
def parse_escape(<<?r, rest::binary>>, pos), do: {"\r", rest, pos + 1}
def parse_escape(<<?v, rest::binary>>, pos), do: {"\v", rest, pos + 1}
def parse_escape(<<??, rest::binary>>, pos), do: {"?", rest, pos + 1}
def parse_escape(<<?0, rest::binary>>, pos), do: {"\0", rest, pos + 1}
def parse_escape(<<?', rest::binary>>, pos), do: {"'", rest, pos + 1}
def parse_escape(<<?", rest::binary>>, pos), do: {"\"", rest, pos + 1}
def parse_escape(<<?/, rest::binary>>, pos), do: {"/", rest, pos + 1}
def parse_escape(<<?\\, rest::binary>>, pos), do: {"\\", rest, pos + 1}
# \xHH
def parse_escape(<<?x, h1, h2, rest::binary>>, pos)
when (h1 in ?0..?9 or h1 in ?a..?f or h1 in ?A..?F) and
(h2 in ?0..?9 or h2 in ?a..?f or h2 in ?A..?F) do
byte = hex_val(h1) * 16 + hex_val(h2)
{<<byte>>, rest, pos + 3}
end
# \uHHHH
def parse_escape(<<?u, h1, h2, h3, h4, rest::binary>>, pos)
when (h1 in ?0..?9 or h1 in ?a..?f or h1 in ?A..?F) and
(h2 in ?0..?9 or h2 in ?a..?f or h2 in ?A..?F) and
(h3 in ?0..?9 or h3 in ?a..?f or h3 in ?A..?F) and
(h4 in ?0..?9 or h4 in ?a..?f or h4 in ?A..?F) do
cp = hex_val(h1) * 0x1000 + hex_val(h2) * 0x100 + hex_val(h3) * 0x10 + hex_val(h4)
# Check for surrogate pair high surrogate
if cp >= 0xD800 and cp <= 0xDBFF do
case rest do
<<?\\, ?u, l1, l2, l3, l4, rest2::binary>>
when (l1 in ?0..?9 or l1 in ?a..?f or l1 in ?A..?F) and
(l2 in ?0..?9 or l2 in ?a..?f or l2 in ?A..?F) and
(l3 in ?0..?9 or l3 in ?a..?f or l3 in ?A..?F) and
(l4 in ?0..?9 or l4 in ?a..?f or l4 in ?A..?F) ->
lo = hex_val(l1) * 0x1000 + hex_val(l2) * 0x100 + hex_val(l3) * 0x10 + hex_val(l4)
if lo >= 0xDC00 and lo <= 0xDFFF do
codepoint = (cp - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000
{<<codepoint::utf8>>, rest2, pos + 12}
else
{:error, "invalid surrogate pair: low surrogate expected"}
end
_ ->
{:error, "incomplete surrogate pair"}
end
else
{<<cp::utf8>>, rest, pos + 5}
end
end
# \UHHHHHHHH
def parse_escape(<<?U, h1, h2, h3, h4, h5, h6, h7, h8, rest::binary>>, pos)
when (h1 in ?0..?9 or h1 in ?a..?f or h1 in ?A..?F) and
(h2 in ?0..?9 or h2 in ?a..?f or h2 in ?A..?F) and
(h3 in ?0..?9 or h3 in ?a..?f or h3 in ?A..?F) and
(h4 in ?0..?9 or h4 in ?a..?f or h4 in ?A..?F) and
(h5 in ?0..?9 or h5 in ?a..?f or h5 in ?A..?F) and
(h6 in ?0..?9 or h6 in ?a..?f or h6 in ?A..?F) and
(h7 in ?0..?9 or h7 in ?a..?f or h7 in ?A..?F) and
(h8 in ?0..?9 or h8 in ?a..?f or h8 in ?A..?F) do
cp =
hex_val(h1) * 0x10000000 + hex_val(h2) * 0x1000000 + hex_val(h3) * 0x100000 +
hex_val(h4) * 0x10000 + hex_val(h5) * 0x1000 + hex_val(h6) * 0x100 +
hex_val(h7) * 0x10 + hex_val(h8)
{<<cp::utf8>>, rest, pos + 9}
end
def parse_escape(<<byte, _rest::binary>>, _pos) do
{:error, "invalid escape sequence: \\#{<<byte>>}"}
end
# Hex digit value
defp hex_val(c) when c in ?0..?9, do: c - ?0
defp hex_val(c) when c in ?a..?f, do: c - ?a + 10
defp hex_val(c) when c in ?A..?F, do: c - ?A + 10
end