toml_elixir

Definitions.

SP = [\s\t]
COMMENT = #[^\n]*

HEX = [0-9a-fA-F]
HEX4 = {HEX}{HEX}{HEX}{HEX}
HEX8 = {HEX}{HEX}{HEX}{HEX}{HEX}{HEX}{HEX}{HEX}
CHAR = [^\\"\x00-\x1f]

ESC_CHAR = \\[btnfr"\\\\]
U4 = \\u{HEX4}
U8 = \\U{HEX8}

BARE_KEY = [a-zA-Z0-9_-]+
LITERAL_STRING = '[^\']*'
BASIC_STRING = "({CHAR}|{ESC_CHAR}|{U4}|{U8})*"
BASIC_STRING_ML = """("?"?({CHAR}|\r?\n|\\{SP}*\r?\n|{ESC_CHAR}|{U4}|{U8}))*(\\{SP}+)?"""

LITERAL_STRING_ML = '''('?'?[^'])*'''



POS_INTEGER = ([0-9]|[1-9](_?[0-9])+)

INTEGER = [+-]?{POS_INTEGER}

KEY_INTEGER = -?{POS_INTEGER}



FRACTION = \.[0-9](_?[0-9])*

EXPONENT = [eE]{INTEGER}

FLOAT = {INTEGER}({FRACTION}|{FRACTION}{EXPONENT}|{EXPONENT})

KEY_FLOAT = -?{POS_INTEGER}[eE]-?{POS_INTEGER}



YEAR  = ([0-9][0-9][0-9][0-9])

MONTH = (0[1-9]|1[0-2])

DAY   = (0[1-9]|[12][0-9]|3[01])

HOUR = ([01][0-9]|2[0-3])

MIN  = ([0-5][0-9])

SEC  = ([0-5][0-9]|60)

DATE = {YEAR}-{MONTH}-{DAY}

TIME = {HOUR}:{MIN}:{SEC}(\.[0-9]+)?

TZOFFSET = (Z|[+-]{HOUR}:{MIN})



Rules.



% NOTE: to tell apart "[[ foo ]]" (valid) and "[ [ foo ] ]" (invalid), spaces

% preceding a bracket need to be reported, but to simplify grammar in general,

% they get skipped in all the other places

{SP}+\[ : {token, {space, TokenLine}, "["}.

{SP}+\] : {token, {space, TokenLine}, "]"}.

\[ : {token, {'[', TokenLine}}.

\] : {token, {']', TokenLine}}.

\{ : {token, {'{', TokenLine}}.

\} : {token, {'}', TokenLine}}.

\. : {token, {'.', TokenLine}}.

=  : {token, {'=', TokenLine}}.

,  : {token, {',', TokenLine}}.

\r?\n : {token, {nl, TokenLine}}.



{DATE}T{TIME}{TZOFFSET} : {token, {datetime_tz, TokenLine, datetime_tz(TokenChars)}}.

{DATE}T{TIME} : {token, {local_datetime, TokenLine, local_datetime(TokenChars)}}.

% local date is a valid bare key

{DATE} : {token, {local_date, TokenLine, {TokenChars, local_date(TokenChars)}}}.

{TIME} : {token, {local_time, TokenLine, local_time(TokenChars)}}.



% these two are valid bare keys

true  : {token, {bool, TokenLine, true}}.

false : {token, {bool, TokenLine, false}}.



{KEY_FLOAT} : {token, {key_float, TokenLine, {TokenChars, to_float(TokenChars)}}}.

{KEY_INTEGER} : {token, {key_integer, TokenLine, {TokenChars, to_integer(TokenChars)}}}.

{FLOAT} : {token, {float, TokenLine, to_float(TokenChars)}}.

{INTEGER} : {token, {integer, TokenLine, to_integer(TokenChars)}}.

{BARE_KEY} : {token, {bare_key, TokenLine, TokenChars}}.



{BASIC_STRING}   : {token, {basic_string, TokenLine, basic_string(TokenChars)}}.

{LITERAL_STRING} : {token, {literal_string, TokenLine, literal_string(TokenChars)}}.

{BASIC_STRING_ML}   : {token, {basic_string_ml, TokenLine, basic_string_ml(TokenChars)}}.

{LITERAL_STRING_ML} : {token, {literal_string_ml, TokenLine, literal_string_ml(TokenChars)}}.



{SP}+     : skip_token.

{COMMENT} : skip_token.



Erlang code.



-export([tokenize/1]).



%% @doc Lexer entry point.



-spec tokenize(string() | binary() | iolist()) ->

  {ok, [term()], integer()} | {error, term(), integer()}.



tokenize(String) ->

  string(unicode:characters_to_list([String, $\n])).



to_integer("+" ++ String) ->

  to_integer(String);

to_integer("-" ++ String) ->

  -to_integer(String);

to_integer(String) ->

  list_to_integer([C || C <- String, C /= $_]).



to_float(String) ->

  StripUnderscore = [C || C <- String, C /= $_],

  case string:chr(StripUnderscore, $.) of

    0 ->

      % fix the float for Erlang's parsing function by inserting the missing
      % fraction part just before "e" marker

      % NOTE: we're here because either "." or "[eE]" was found in the number,

      % and we know the "." is missing

      [A, B] = string:tokens(StripUnderscore, "eE"),
      list_to_float(A ++ ".0e" ++ B);
    _ ->
      list_to_float(StripUnderscore)
  end.

datetime_tz(String) ->
  StrLen = length(String),
  case lists:last(String) of
    $Z ->
      Timezone = "Z",
      {local_datetime(string:substr(String, 1, StrLen - 1)), Timezone};
    _ ->
      Timezone = string:substr(String, 1 + StrLen - 6, 6),
      {local_datetime(string:substr(String, 1, StrLen - 6)), Timezone}
  end.

local_datetime(String) ->
  [Date, Time] = string:tokens(String, "T"),
  {local_date(Date), local_time(Time)}.

local_date(String) ->
  [Y, M, D] = string:tokens(String, "-"),
  {list_to_integer(Y), list_to_integer(M), list_to_integer(D)}.

local_time(String) ->
  [HH, MM, SS] = string:tokens(String, ":"),
  case string:tokens(SS, ".") of
    [_] ->
      {list_to_integer(HH), list_to_integer(MM), list_to_integer(SS)};
    [S, Frac] ->
      {{list_to_integer(HH), list_to_integer(MM), list_to_integer(S)}, Frac}
  end.

literal_string(String) ->
  string:substr(String, 2, length(String) - 2).

literal_string_ml(String) ->
  case string:substr(String, 4, length(String) - 6) of
    "\r\n" ++ Result -> Result;
    "\n" ++ Result -> Result;
    Result -> Result
  end.

basic_string(String) ->
  escape(string:substr(String, 2, length(String) - 2)).

basic_string_ml(String) ->
  case string:substr(String, 4, length(String) - 6) of
    "\r\n" ++ Result -> escape(Result);
    "\n" ++ Result -> escape(Result);
    Result -> escape(Result)
  end.

escape("") -> "";
escape("\\b"  ++ Rest) -> [$\b | escape(Rest)];
escape("\\t"  ++ Rest) -> [$\t | escape(Rest)];
escape("\\n"  ++ Rest) -> [$\n | escape(Rest)];
escape("\\f"  ++ Rest) -> [$\f | escape(Rest)];
escape("\\r"  ++ Rest) -> [$\r | escape(Rest)];
escape("\\\"" ++ Rest) -> [$\" | escape(Rest)];
escape("\\\\" ++ Rest) -> [$\\ | escape(Rest)];
escape("\\u"  ++ [C1, C2, C3, C4 | Rest]) ->
  [u([C4, C3, C2, C1]) | escape(Rest)];
escape("\\U"  ++ [C1, C2, C3, C4, C5, C6, C7, C8 | Rest]) ->
  [u([C8, C7, C6, C5, C4, C3, C2, C1]) | escape(Rest)];
escape("\\" ++ Rest) ->
  escape(skip_spaces(Rest));
escape([C | Rest]) ->
  [C | escape(Rest)].

skip_spaces(" " ++ Rest) -> skip_spaces(Rest);
skip_spaces("\t" ++ Rest) -> skip_spaces(Rest);
skip_spaces("\n" ++ Rest) -> skip_spaces(Rest);
skip_spaces("\r\n" ++ Rest) -> skip_spaces(Rest);
skip_spaces(String) -> String.

u([]) -> 0;
u([C | Rest]) when C >= $0 andalso C =< $9 ->      (C - $0) + 16 * u(Rest);
u([C | Rest]) when C >= $A andalso C =< $F -> (10 + C - $A) + 16 * u(Rest);
u([C | Rest]) when C >= $a andalso C =< $f -> (10 + C - $a) + 16 * u(Rest).