Packages
gpb
5.0.0
5.0.0
4.21.7
4.21.6
4.21.5
4.21.4
4.21.3
4.21.2
4.21.1
4.21.0
4.20.0
4.19.9
4.19.8
4.19.7
4.19.6
4.19.5
4.19.4
4.19.3
4.19.2
4.19.1
4.19.0
4.18.0
4.17.7
4.17.6
4.17.5
4.17.3
4.17.2
4.17.1
4.17.0
4.16.2
4.16.1
4.16.0
4.15.2
4.15.1
4.14.2
4.14.1
4.14.0
4.13.0
4.12.0
4.11.2
4.11.1
4.11.0
4.10.6
4.10.5
4.10.4
4.10.3
4.10.2
4.10.1
4.10.0
4.9.3
4.9.2
4.9.1
4.9.0
4.8.0
4.7.3
4.7.2
4.7.1
4.7.0
4.6.0
4.5.1
4.5.0
4.4.1
4.4.0
4.3.3
4.3.2
4.3.1
4.3.0
4.2.3
4.2.2
4.2.1
4.2.0
4.1.9
4.1.8
4.1.7
4.1.6
4.1.5
4.1.4
4.1.3
4.1.2
4.1.1
4.1.0
4.0.2
4.0.1
4.0.0
3.28.1
3.28.0
3.27.7
3.27.6
3.27.5
3.27.4
3.27.3
3.27.2
3.27.1
3.27.0
3.26.8
3.26.7
3.26.6
3.26.5
3.26.4
3.26.3
3.26.2
3.26.1
3.26.0
3.25.2
3.25.1
3.25.0
3.24.4
3.24.3
3.24.2
3.24.1
3.24.0
3.23.2
3.23.1
3.23.0
3.22.5
3.22.4
3.22.3
3.22.2
3.22.1
3.22.0
3.21.3
3.21.2
3.21.1
3.21.0
3.20.3
3.20.2
3.20.0
3.19.0
3.18.10
3.18.9
3.18.8
3.18.7
3.18.6
3.18.5
3.18.4
3.18.3
3.18.2
3.18.1
3.18.0
3.17.13
3.17.12
3.17.11
3.17.10
3.17.9
3.17.8
3.17.5
3.17.4
3.17.3
3.17.2
3.17.1
3.17.0
3.16.0
3.15.0
3.14.0
3.13.0
3.12.2
3.12.1
3.12.0
3.11.0
A compiler for Google protocol buffer definitions files for Erlang.
Current section
Files
Jump to
Current section
Files
src/gpb_parse.erl
%%% Copyright (C) 2019 Tomas Abrahamsson
%%%
%%% Author: Tomas Abrahamsson <tab@lysator.liu.se>
%%%
%%% This library is free software; you can redistribute it and/or
%%% modify it under the terms of the GNU Lesser General Public
%%% License as published by the Free Software Foundation; either
%%% version 2.1 of the License, or (at your option) any later version.
%%%
%%% This library is distributed in the hope that it will be useful,
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
%%% Lesser General Public License for more details.
%%%
%%% You should have received a copy of the GNU Lesser General Public
%%% License along with this library; if not, write to the Free Software
%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
%%% MA 02110-1301 USA
%% @doc This is the parser
%% @private
-module(gpb_parse).
-export([parse/1]).
-export([format_error/1]).
-export_type([error/0]).
-type error() :: {Line::pos_integer(), module(), Reason::term()}.
%% -- bwd compat -- do not use these ----------------------------------
%% Deprecated!
%% Instead, to retrieve proto definitions,
%% use the option to_proto_defs
%% with gpb_compile:file or gpb_compile:string.
%% For exprotobuf, see also
%% https://github.com/bitwalker/exprotobuf/issues/114
-export([post_process_one_file/3]). % use opt to_proto_defs instead
-export([post_process_all_files/2]). % use opt to_proto_defs instead
-export([format_post_process_error/1]). % use gpb_compile:format_error/1
-export([fetch_imports/1]). % use gpb_defs:fetch_imports
%% --^^--- bwd compat -- do not use these ----------------------------
-include("../include/gpb.hrl").
%% @doc Parse a list of tokens as returned from {@link gpb_parse2:binary/1}
%%
%% Please, do not use this from outside of gpb. Instead, use the option
%% `to_proto_defs' with gpb_compile:file/1,2 or gob_string/2,3, to get
%% a parsed .proto file.
%%
%% @hidden
-spec parse([gpb_scan:token()]) -> {ok, gpb_defs:defs()} | {error, [error()]}.
parse(Tokens) ->
{ParseTree, Errors} = p_top(Tokens, [], []),
if Errors == [] -> {ok, ParseTree};
Errors /= [] -> {error, Errors}
end.
-define(f(Fmt, Args), io_lib:format(Fmt, Args)).
%% @doc Format an error. Note that the {@link parse/1} can return a list of
%% errors. This function formats one element in such a list.
format_error({syntax_error, {before, Tokens}}) ->
?f("syntax error at: ~s", [tokens_to_str(Tokens)]);
format_error({syntax_error, {before, Tokens}, Why}) ->
?f("syntax error at: ~s: ~s", [tokens_to_str(Tokens), ensure_str(Why)]).
-define(t(Token), {Token, _, _}). % a token
-define(w(N), ?t(<<N>>)). % a word, as a binary
-define(s(S), ?t({str_lit, S})). % a string literal
-define(i(I), ?t({int_lit, I})). % an integer literal
-define(fl(F), ?t({float_lit, F})). % a float literal
-define(syntax_error(Where), throw({syntax_error, line(Where), Where})).
-define(syntax_error(Where, Why),
throw({syntax_error, line(Where), Where, Why})).
%% Principles that most of the recursive descent parser below follow:
%%
%% * To parse each an item, there is a corresponding function p_item.
%% * Each p_item either returns {Item, Rest} or fail with a syntax error.
%% * Each p_item parses an entire <item>, including the first token,
%% even when some caller peeked to know which p_<item> to call.
%% This is to make sub routines more reusable.
p_top(Tokens, Acc, Errors) when Tokens /= [] ->
try
case hd(Tokens) of
?w("syntax") ->
{Syntax, Rest} = p_syntax(Tokens),
Acc1 = [Syntax | Acc],
p_top(Rest, Acc1, Errors);
?w("package") ->
{Package, Rest} = p_package(Tokens),
Acc1 = [Package | Acc],
p_top(Rest, Acc1, Errors);
?w("import") ->
{Import, Rest} = p_import(Tokens),
Acc1 = [Import | Acc],
p_top(Rest, Acc1, Errors);
?w("enum") ->
{Enum, Rest} = p_enum(Tokens),
Acc1 = [Enum | Acc],
p_top(Rest, Acc1, Errors);
?w("message") ->
{Msg, Rest} = p_message(Tokens),
Acc1 = [Msg | Acc],
p_top(Rest, Acc1, Errors);
?w("extend") ->
{Extend, Rest} = p_extend(Tokens),
Acc1 = [Extend | Acc],
p_top(Rest, Acc1, Errors);
?w("option") ->
{Option, Rest} = p_option(Tokens),
Acc1 = [Option | Acc],
p_top(Rest, Acc1, Errors);
?w("service") ->
{Service, Rest} = p_service(Tokens),
Acc1 = [Service | Acc],
p_top(Rest, Acc1, Errors);
?t(';') ->
p_top(tl(Tokens), Acc, Errors);
{'$end', _Line} ->
%% bwd compat with the old parser
p_top([], Acc, Errors);
_ ->
?syntax_error(Tokens)
end
catch throw:{syntax_error, Line, FollowingTokens}:St ->
maybe_debug_syntax_error(Line, FollowingTokens, St, undefined),
ParenStack = [],
Rest1 = try_recover(ParenStack, safe_tl(Tokens)),
Rest2 = skip_semicolon(Rest1),
Where = safe_max_n_on_same_line(FollowingTokens, 3),
Error = {Line, ?MODULE, {syntax_error, {before, Where}}},
p_top(Rest2, Acc, [Error | Errors]);
throw:{syntax_error, Line, FollowingTokens, Why}:St ->
maybe_debug_syntax_error(Line, FollowingTokens, St, Why),
ParenStack = [],
Rest1 = try_recover(ParenStack, safe_tl(Tokens)),
Rest2 = skip_semicolon(Rest1),
Where = safe_max_n_on_same_line(FollowingTokens, 3),
Error = {Line, ?MODULE, {syntax_error, {before, Where}, Why}},
p_top(Rest2, Acc, [Error | Errors])
end;
p_top([], Acc, Errors) ->
{lists:reverse(Acc), lists:reverse(Errors)}.
maybe_debug_syntax_error(Line, FollowingTokens, StackTrace, Why) ->
case os:getenv("GPB_DEBUG_PARSER") of
Yes when Yes == "1"; Yes == "true" ->
io:format("Syntax error on line ~p~n ~P~n ~p~n ~p~n",
[Line, FollowingTokens, 10, Why, StackTrace]);
_ ->
ok
end.
safe_tl([_ | Rest]) -> Rest;
safe_tl([]) -> [].
line([{_Token, Line, _Orig} | _]) -> Line;
line([]) -> 'at end-of-file'.
safe_max_n_on_same_line([], _Max) -> [];
safe_max_n_on_same_line([{_, Line, _Orig} | _]=Tokens, Max) ->
safe_max_aux(Tokens, Max, Line).
safe_max_aux([{_, Line, _Orig}=Token | Rest], Max, Line) when Max >= 1 ->
[Token | safe_max_aux(Rest, Max-1, Line)];
safe_max_aux(_, _Max, _Line) ->
[].
try_recover([], [?t(';') | Rest]) -> Rest;
%% Top-level items
try_recover([], [?w("package") | _]=Rest) -> Rest;
try_recover([], [?w("message") | _]=Rest) -> Rest;
try_recover([], [?w("extend") | _]=Rest) -> Rest;
try_recover([], [?w("option") | _]=Rest) -> Rest;
try_recover([], [?w("service") | _]=Rest) -> Rest;
%% Push parentheses
try_recover(PStk, [?t('(') | Rest]) -> try_recover(['(' | PStk], Rest);
try_recover(PStk, [?t('{') | Rest]) -> try_recover(['{' | PStk], Rest);
try_recover(PStk, [?t('[') | Rest]) -> try_recover(['[' | PStk], Rest);
%% Pop parentheses
try_recover(['{'], [?t('}') | Rest]) -> Rest; % top-level
try_recover(['(' | PStk], [?t(')') | Rest]) -> try_recover(PStk, Rest);
try_recover(['{' | PStk], [?t('}') | Rest]) -> try_recover(PStk, Rest);
try_recover(['[' | PStk], [?t(']') | Rest]) -> try_recover(PStk, Rest);
try_recover(PStk, [_ | Rest]) -> try_recover(PStk, Rest);
try_recover(_PStk, []) -> [].
%% --------------------------------------------
%% syntax
%% syntax_def -> syntax '=' str_lit ';'
p_syntax(Tokens) ->
case Tokens of
[?w("syntax"), ?t('=') | Rest] ->
{Value, Rest2} = p_const(Rest),
Rest3 = skip_semicolon(Rest2),
case verify_syntax(Value) of
ok ->
{{syntax, Value}, Rest3};
{error, Why} ->
?syntax_error(Rest, Why)
end;
_ ->
?syntax_error(Tokens)
end.
%% --------------------------------------------
%% package
%% package_def -> package dotted_name ';'
p_package([?w("package") | Rest]) ->
{Name, Rest2} = p_dotted_name(Rest),
Rest3 = skip_semicolon(Rest2),
{{package, Name}, Rest3}.
%% --------------------------------------------
%% import
%% import_def -> import str_lit ';'
p_import([?w("import") | Rest]) ->
case Rest of
[?s(Import) | Rest2] ->
Rest3 = skip_semicolon(Rest2),
{{import, str_value(Import)}, Rest3};
[?w("public"), ?s(Import) | Rest2] ->
Rest3 = skip_semicolon(Rest2),
{{import, str_value(Import)}, Rest3};
[?w("weak"), ?s(Import) | Rest2] ->
Rest3 = skip_semicolon(Rest2),
{{import, str_value(Import)}, Rest3};
_ ->
?syntax_error(Rest)
end;
p_import(Tokens) ->
?syntax_error(Tokens, "expected import <string>;").
%% --------------------------------------------
%% enum
%% enum_def -> enum name '{' enum_fields '}'
%%
%% enum_fields -> enum_field enum_fields
%% enum_fields -> option_def enum_fields
%% enum_fields -> ';' enum_fields
%% enum_fields -> '$empty'
%%
%% enum_field -> identifier '=' integer ';'
%% enum_field -> identifier '=' integer '[' opt_list ']' ';'
%%
p_enum([?w("enum"), ?w(Name/binary), ?t('{') | Rest]) ->
Rest2 = skip_semicolon(Rest),
{EnumItems, [?t('}') | Rest3]} = p_enum_fields(Rest2, []),
Rest4 = skip_semicolon(Rest3),
{{{enum, word_value(Name)}, EnumItems}, Rest4};
p_enum(Tokens) ->
?syntax_error(Tokens, "expected enum <name> { <symbol> = <value>; ... }").
p_enum_fields(Tokens, Acc) ->
case Tokens of
[?w("option") | _] ->
{Option, Rest} = p_option(Tokens),
p_enum_fields(Rest, [{Option} | Acc]);
[?w("reserved") | _] ->
{Reserved, Rest} = p_reserved(Tokens),
p_enum_fields(Rest, [Reserved | Acc]);
[?w(Name/binary), ?t('=') | Rest] ->
{Value, Rest2} = p_integer_const(Rest),
{EOpts, Rest3} = p_maybe_opt_list(Rest2),
EnumField = {word_value(Name), Value, EOpts},
Rest4 = skip_semicolon(Rest3),
Acc1 = [EnumField | Acc],
p_enum_fields(Rest4, Acc1);
[?t('}') | _] ->
{lists:reverse(Acc), Tokens};
_ ->
?syntax_error(Tokens, "expected <symbol> = <value>;")
end.
%% --------------------------------------------
%% message
%% message_def -> message fidentifier '{' msg_elems '}'
%%
%% msg_elems -> msg_elem msg_elems
%% msg_elems -> ';' msg_elems
%% msg_elems -> '$empty'
%%
%% msg_elem -> occurrence type identifier '=' dec_lit ';'
%% msg_elem -> occurrence type identifier '=' dec_lit '[' opt_list ']' ';'
%% msg_elem -> type identifier '=' dec_lit ';' % proto3
%% msg_elem -> type identifier '=' dec_lit '[' opt_list ']' ';' % proto3
%% msg_elem -> map_type identifier '=' dec_lit ';'
%% msg_elem -> map_type identifier '=' dec_lit '[' opt_list ']' ';'
%% msg_elem -> message_def
%% msg_elem -> enum_def
%% msg_elem -> extensions_def
%% msg_elem -> oneof_def
%% msg_elem -> extend_def
%% msg_elem -> reserved_def
%% msg_elem -> group_def
%% msg_elem -> option_def
%%
%% occurrence -> required
%% occurrence -> optional
%% occurrence -> repeated
%%
%% type -> double
%% type -> float
%% type -> int32
%% type -> int64
%% type -> uint32
%% type -> uint64
%% type -> sint32
%% type -> sint64
%% type -> fixed32
%% type -> fixed64
%% type -> sfixed32
%% type -> sfixed64
%% type -> bool
%% type -> string
%% type -> bytes
%% type -> name
%%
p_message([?w("message"), ?w(Name/binary), ?t('{') | Rest]) ->
Rest2 = skip_semicolon(Rest),
{MsgElems, Rest3} = p_msg_elems(Rest2, []),
Msg = {{msg, word_value(Name)}, MsgElems},
Rest4 = skip_semicolon(Rest3),
{Msg, Rest4};
p_message(Tokens) ->
ExpectedWhat = "expected message <name> { <fields, messages or enums> }",
?syntax_error(Tokens, ExpectedWhat).
p_msg_elems(Tokens, Acc) ->
case Tokens of
[?t(';') | Rest] ->
p_msg_elems(Rest, Acc);
[?t('}') | Rest] ->
{lists:reverse(Acc), Rest};
_ ->
{MsgElem, Rest} = p_msg_elem(Tokens),
Rest2 = skip_semicolon(Rest),
p_msg_elems(Rest2, [MsgElem | Acc])
end.
p_msg_elem(Tokens) ->
case hd(Tokens) of
?w("message") ->
p_message(Tokens);
?w("enum") ->
p_enum(Tokens);
?w("extensions") ->
p_extensions(Tokens);
?w("oneof") ->
p_oneof(Tokens);
?w("extend") ->
p_extend(Tokens);
?w("reserved") ->
p_reserved(Tokens);
?w("option") ->
{Option, Rest} = p_option(Tokens),
{{Option}, Rest};
?w("map") ->
p_map(Tokens);
?w("required") ->
p_field_or_group(required, tl(Tokens));
?w("optional") ->
p_field_or_group(optional, tl(Tokens));
?w("repeated") ->
p_field_or_group(repeated, tl(Tokens));
_ ->
p_field(undefined, Tokens)
end.
%% group_def -> occurrence group identifier '=' dec_lit '{' msg_elems '}':
%%
p_field_or_group(Occurrence, [?w("group"), ?w(Name/binary), ?t('=') | Rest]) ->
{FNum, Rest2} = p_integer_const(Rest),
{_Opts, Rest3} = p_maybe_opt_list(Rest2),
case Rest3 of
[?t('{') | Rest4] ->
Rest5 = skip_semicolon(Rest4),
{MsgElems, Rest6} = p_msg_elems(Rest5, []),
TmpGName = word_value(Name),
Field = #?gpb_field{occurrence = Occurrence,
type = {ref,['...expanded-later']},
name = TmpGName,
fnum = FNum,
opts = []},
Group = {group1, TmpGName, MsgElems, Field},
Rest7 = skip_semicolon(Rest6),
{Group, Rest7};
_ ->
?syntax_error(Rest2)
end;
p_field_or_group(Occurrence, Tokens) ->
p_field(Occurrence, Tokens).
p_field(Occurrence, Tokens) ->
{Type, Rest} = p_field_type(Tokens),
case Rest of
[?w(FName/binary), ?t('=') | Rest2] ->
{FNum, Rest3} = p_integer_const(Rest2),
{FOpts, Rest4} = p_field_opts(Rest3),
Field = #?gpb_field{name = word_value(FName),
type = Type,
occurrence = Occurrence,
fnum = FNum,
opts = FOpts},
Rest5 = skip_semicolon(Rest4),
{Field, Rest5};
_ ->
?syntax_error(Rest, "expected <field> = <num>")
end.
p_field_type([?w("double") | Rest]) -> {double, Rest};
p_field_type([?w("float") | Rest]) -> {float, Rest};
p_field_type([?w("int32") | Rest]) -> {int32, Rest};
p_field_type([?w("int64") | Rest]) -> {int64, Rest};
p_field_type([?w("uint32") | Rest]) -> {uint32, Rest};
p_field_type([?w("uint64") | Rest]) -> {uint64, Rest};
p_field_type([?w("sint32") | Rest]) -> {sint32, Rest};
p_field_type([?w("sint64") | Rest]) -> {sint64, Rest};
p_field_type([?w("fixed32") | Rest]) -> {fixed32, Rest};
p_field_type([?w("fixed64") | Rest]) -> {fixed64, Rest};
p_field_type([?w("sfixed32") | Rest]) -> {sfixed32, Rest};
p_field_type([?w("sfixed64") | Rest]) -> {sfixed64, Rest};
p_field_type([?w("bool") | Rest]) -> {bool, Rest};
p_field_type([?w("string") | Rest]) -> {string, Rest};
p_field_type([?w("bytes") | Rest]) -> {bytes, Rest};
p_field_type(Tokens) ->
{Name, Rest} = p_dotted_name(Tokens),
{{ref, Name}, Rest}.
p_field_opts(Tokens) ->
{Opts, Rest} = p_maybe_opt_list(Tokens),
{[normalize_field_opt(Opt) || Opt <- Opts], Rest}.
normalize_field_opt({_,_}=Opt) -> Opt;
normalize_field_opt(Opt) -> {Opt, true}.
%% map_type -> map '<' map_key_type ',' type '>'
%%
%% map_key_type -> int32
%% map_key_type -> int64
%% map_key_type -> uint32
%% map_key_type -> uint64
%% map_key_type -> sint32
%% map_key_type -> sint64
%% map_key_type -> fixed32
%% map_key_type -> fixed64
%% map_key_type -> sfixed32
%% map_key_type -> sfixed64
%% map_key_type -> bool
%% map_key_type -> string
%% %% missing from type: double | float | bytes | message name | enum name
%%
p_map([?w("map"), ?t('<') | Rest]) ->
{KeyType, Rest2} = p_map_key_type(Rest),
case Rest2 of
[?t(',') | Rest3] ->
{ValueType, Rest4} = p_field_type(Rest3),
case Rest4 of
[?t('>'), ?w(FName/binary), ?t('=') | Rest5] ->
{FNum, Rest6} = p_integer_const(Rest5),
Type = {map, KeyType, ValueType},
{FOpts, Rest7} = p_field_opts(Rest6),
Field = #?gpb_field{name = word_value(FName),
type = Type,
occurrence = repeated,
fnum = FNum,
opts = FOpts},
Rest8 = skip_semicolon(Rest7),
{Field, Rest8};
_ ->
?syntax_error(Rest4, expected_mapfield_tokens())
end;
_->
?syntax_error(Rest2, expected_mapfield_tokens())
end;
p_map(Tokens) ->
?syntax_error(Tokens, expected_mapfield_tokens()).
expected_mapfield_tokens() ->
"expected map< <key type>, <value type> > = <num>;".
p_map_key_type([?w("int32") | Rest]) -> {int32, Rest};
p_map_key_type([?w("int64") | Rest]) -> {int64, Rest};
p_map_key_type([?w("uint32") | Rest]) -> {uint32, Rest};
p_map_key_type([?w("uint64") | Rest]) -> {uint64, Rest};
p_map_key_type([?w("sint32") | Rest]) -> {sint32, Rest};
p_map_key_type([?w("sint64") | Rest]) -> {sint64, Rest};
p_map_key_type([?w("fixed32") | Rest]) -> {fixed32, Rest};
p_map_key_type([?w("fixed64") | Rest]) -> {fixed64, Rest};
p_map_key_type([?w("sfixed32") | Rest]) -> {sfixed32, Rest};
p_map_key_type([?w("sfixed64") | Rest]) -> {sfixed64, Rest};
p_map_key_type([?w("bool") | Rest]) -> {bool, Rest};
p_map_key_type([?w("string") | Rest]) -> {string, Rest}.
%% oneof_def -> 'oneof' identifier '{' oneof_elems '}'
%%
%% oneof_elems -> oneof_elem oneof_elems
%% oneof_elems -> oneof_elem
%%
%% oneof_elem -> type fidentifier '=' dec_lit ';'
%% oneof_elem -> type fidentifier '=' dec_lit '[' opt_list ']' ';'
p_oneof([?w("oneof"), ?w(Name/binary), ?t('{') | Rest]) ->
Rest2 = skip_semicolon(Rest),
{Elems, [?t('}') | Rest3]} = p_oneof_elems(Rest2, []),
{Opts, OFields} = lists:partition(
fun({{option, _OptName, _OptValue}}) -> true;
(_Other) -> false
end,
Elems),
Opts1 = [{OptName,OptVal} || {{option, OptName, OptVal}} <- Opts],
Field = #gpb_oneof{name = word_value(Name),
fields = OFields,
opts = Opts1},
Rest4 = skip_semicolon(Rest3),
{Field, Rest4};
p_oneof(Tokens) ->
?syntax_error(Tokens, "expected oneof <name> { <fields> }").
p_oneof_elems(Tokens, Acc) ->
case Tokens of
[?w("option") | _] ->
{Opt, Rest} = p_option(Tokens),
Rest2 = skip_semicolon(Rest),
Acc1 = [{Opt} | Acc],
p_oneof_elems(Rest2, Acc1);
_ ->
{Field, Rest} = p_field_or_group(optional, Tokens),
Rest2 = skip_semicolon(Rest),
Acc1 = [Field | Acc],
case Rest2 of
[?t('}') | _] ->
{lists:reverse(Acc1), Rest2};
_ ->
p_oneof_elems(Rest2, Acc1)
end
end.
%% extensions_def -> extensions exts ';'
%%
%% exts -> ext ',' exts
%% exts -> ext
%%
%% ext -> integer
%% ext -> integer to integer
%% ext -> integer to max
p_extensions([?w("extensions") | Rest]) ->
{Exts, Rest2} = p_exts(Rest, []),
{_Opts, Rest3} = p_maybe_opt_list(Rest2),
Rest4 = skip_semicolon(Rest3),
{{extensions, lists:sort(Exts)}, Rest4}.
p_exts(Tokens, Acc) ->
{Ext, Rest} = p_ext(Tokens),
Acc1 = [Ext | Acc],
case Rest of
[?t(',') | Rest2] ->
p_exts(Rest2, Acc1);
_ ->
{lists:reverse(Acc1), Rest}
end.
p_ext([?i(Min), ?w("to"), ?i(Max) | Rest]) ->
{{int_value(Min), int_value(Max)}, Rest};
p_ext([?i(Min), ?w("to"), ?w("max") | Rest]) ->
{{int_value(Min), max}, Rest};
p_ext([?i(Int) | Rest]) ->
I = int_value(Int),
{{I, I}, Rest};
p_ext(Tokens) ->
?syntax_error(Tokens, "expected <n>, <n> to <m>, or <n> to max").
%% reserved_def -> reserved res_numbers
%% reserved_def -> reserved res_names
%%
%% res_numbers -> res_number ',' res_numbers
%% res_numbers -> res_number
%%
%% res_number -> integer
%% res_number -> integer to integer
%%
%% res_names -> string_expr ',' res_names
%% res_names -> string_expr
%%
p_reserved([?w("reserved") | Rest]) ->
case hd(Rest) of
?i(_) ->
{Numbers, Rest2} = p_reserved_numbers_or_ranges(Rest, []),
Rest3 = skip_semicolon(Rest2),
{{reserved_numbers, Numbers}, Rest3};
?t('-') ->
{Numbers, Rest2} = p_reserved_numbers_or_ranges(Rest, []),
Rest3 = skip_semicolon(Rest2),
{{reserved_numbers, Numbers}, Rest3};
?s(_) ->
{Names, Rest2} = p_reserved_names(Rest, []),
Rest3 = skip_semicolon(Rest2),
{{reserved_names, Names}, Rest3};
_ ->
?syntax_error(Rest, "expected reserved numbers or ranges or names")
end.
p_reserved_numbers_or_ranges(Tokens, Acc) ->
{Reserved, Rest} = p_reserved_number_or_range(Tokens),
Acc1 = [Reserved | Acc],
case Rest of
[?t(',') | Rest2] ->
p_reserved_numbers_or_ranges(Rest2, Acc1);
_ ->
{lists:reverse(Acc1), Rest}
end.
p_reserved_number_or_range(Tokens) ->
{Min, Rest} = p_integer_const(Tokens),
case Rest of
[?w("to"), ?w("max") | Rest2] ->
{{Min, max}, Rest2};
[?w("to") | Rest2] ->
{Max, Rest3} = p_integer_const(Rest2),
{{Min, Max}, Rest3};
_ ->
{Min, Rest}
end.
p_reserved_names(Tokens, Acc) ->
{Reserved, Rest} = p_reserved_name(Tokens),
Acc1 = [Reserved | Acc],
case Rest of
[?t(',') | Rest2] ->
p_reserved_names(Rest2, Acc1);
_ ->
{lists:reverse(Acc1), Rest}
end.
p_reserved_name(Tokens) ->
p_str_const(Tokens, []).
%% --------------------------------------------
%% extend
%% extend_def -> extend name '{' msg_elems '}':
p_extend([?w("extend") | Rest]) ->
{Name, Rest2} = p_dotted_name(Rest),
case Rest2 of
[?t('{') | Rest3] ->
Rest4 = skip_semicolon(Rest3),
{MsgElems, Rest5} = p_msg_elems(Rest4, []),
Extend = {{extend, {eref1,Name}}, MsgElems},
Rest6 = skip_semicolon(Rest5),
{Extend, Rest6};
_ ->
?syntax_error(Rest2)
end.
%% --------------------------------------------
%% option
%% option_def -> option option_name '=' constant
p_option([?w("option") | Rest]=Tokens) ->
case p_option_name(Rest) of
{OptName, [?t('=') | Rest2]} ->
{Value, Rest3} = p_option_value(Rest2),
Rest4 = skip_semicolon(Rest3),
{{option, OptName, Value}, Rest4};
_ ->
?syntax_error(Tokens, "expected option <name> = <value>")
end;
p_option(Tokens) ->
?syntax_error(Tokens).
p_option_value([?t('{')=T | Rest]) ->
p_uninterpreted_block(Rest, 1, [T]);
p_option_value(Tokens) ->
p_const(Tokens).
p_uninterpreted_block([Token | Rest], Depth, Acc) ->
%% Just count curly braces until we find a matching one.
%% This seems to be what the protobuf does.
case Token of
?t('}') ->
if Depth =:= 1 ->
AccTokens = lists:reverse([Token | Acc]),
S = lists:flatten(tokens_to_str(AccTokens)),
V = {uninterpreted, S},
{V, Rest};
Depth > 1 ->
p_uninterpreted_block(Rest, Depth-1, [Token | Acc])
end;
?t('{') ->
p_uninterpreted_block(Rest, Depth+1, [Token | Acc]);
_ ->
p_uninterpreted_block(Rest, Depth, [Token | Acc])
end;
p_uninterpreted_block([]=Tokens, _Depth, Acc) ->
L0Str = integer_to_list(line(lists:last(Acc))),
Why = "unexpected end of input in option block starting at line " ++ L0Str,
?syntax_error(Tokens, Why).
%% --------------------------------------------
%% service
%% service_def -> service fidentifier '{' rpc_defs '}'
%%
%% rpc_defs -> rpc_def rpc_defs
%% rpc_defs -> ';' rpc_defs
%% rpc_defs -> '$empty'
%%
%% rpc_def -> rpc fidentifier rpc_arg returns rpc_ret ';':
%% rpc_def -> rpc fidentifier rpc_arg returns rpc_ret '{' m_opts '}':
%%
%% rpc_arg -> '(' name ')'
%% rpc_arg -> '(' stream name ')'
%%
%% rpc_ret -> '(' name ')'
%% rpc_ret -> '(' stream name ')'
%%
%% m_opts -> option_def ';' m_opts
%% m_opts -> ';' m_opts
%% m_opts -> '$empty'
p_service([?w("service"), ?w(Name/binary), ?t('{') | Rest]) ->
Rest2 = skip_semicolon(Rest),
{RpcDefs, [?t('}') | Rest3]} = p_rpc_defs(Rest2, []),
Service = {{service, word_value(Name)}, RpcDefs},
Rest4 = skip_semicolon(Rest3),
{Service, Rest4}.
p_rpc_defs(Tokens, Acc) ->
case Tokens of
[?t('}') | _] ->
{lists:reverse(Acc), Tokens};
[?w("option") | _] ->
{Opt, Rest} = p_option(Tokens),
Acc1 = [{Opt} | Acc],
p_rpc_defs(Rest, Acc1);
[?w("rpc") | _] ->
{Rpc, Rest} = p_rpc_def(Tokens),
Acc1 = [Rpc | Acc],
Rest2 = skip_semicolon(Rest),
p_rpc_defs(Rest2, Acc1);
_ ->
?syntax_error(Tokens, "expected rpc definitions")
end.
p_rpc_def([?w("rpc"), ?w(Name/binary) | Rest]) ->
case p_rpc_arg(Rest) of
{RpcArg, [?w("returns") | Rest2]} ->
{RpcRet, Rest3} = p_rpc_ret(Rest2),
{MOpts, Rest4} = p_maybe_rpc_method_opts(Rest3),
Rpc = {word_value(Name), RpcArg, RpcRet, MOpts},
Rest5 = skip_semicolon(Rest4),
{Rpc, Rest5};
_ ->
?syntax_error(Rest, "expected returns")
end.
p_rpc_arg(Tokens) -> p_rpc_ar(Tokens).
p_rpc_ret(Tokens) -> p_rpc_ar(Tokens).
p_rpc_ar([?t('(') | Rest]) ->
{IsStream, Rest3} =
case Rest of
[?w("stream") | Rest2] ->
{true, Rest2};
_ ->
{false, Rest}
end,
case p_dotted_name(Rest3) of
{Name, [?t(')') | Rest4]} ->
{{Name, IsStream}, Rest4};
_ ->
?syntax_error(Rest3)
end;
p_rpc_ar(Tokens) ->
?syntax_error(Tokens).
p_maybe_rpc_method_opts([?t('{') | Rest]) ->
Rest2 = skip_semicolon(Rest),
{Opts, [?t('}') | Rest3]} = p_rpc_method_opts(Rest2, []),
{Opts, Rest3};
p_maybe_rpc_method_opts(Tokens) ->
{[], Tokens}.
p_rpc_method_opts(Tokens, Acc) ->
case Tokens of
[?t('}') | _] ->
{lists:reverse(Acc), Tokens};
[?w("option") | _] ->
{Option, Rest} = p_option(Tokens),
Acc1 = [Option | Acc],
Rest2 = skip_semicolon(Rest),
p_rpc_method_opts(Rest2, Acc1);
_ ->
?syntax_error(Tokens, "expected rpc method options")
end.
%% --------------------------------------------
%% misc common
%% dotted_name -> '.' name_parts
%% dotted_name -> name_parts
%%
%% name_parts -> name '.' name_parts
%% name_parts -> name
p_dotted_name([?t('.') | Rest]) -> p_dn2(Rest, ['.']);
p_dotted_name(Tokens) -> p_dn2(Tokens, []).
p_dn2([?w(Name/binary) | Rest], Acc) ->
p_dn3(Rest, [word_value(Name) | Acc]);
p_dn2(Tokens, _Acc) ->
?syntax_error(Tokens, "expected dotted name").
p_dn3([?t('.'), ?w(Name/binary) | Tl], NAcc) ->
p_dn3(Tl, [word_value(Name), '.' | NAcc]);
p_dn3(Tokens, Acc) ->
{lists:reverse(Acc), Tokens}.
%% option_name -> option_name_part { '.' option_name_part }*
%%
%% option_name_part -> identifier
%% option_name_part -> '(' dotted_name ')'
%%
%% Return: an atom if the option_name is a single identifier
%% a flat list if the option is any form of dotted name
p_option_name(Tokens) ->
{NamePart, Rest} = p_option_name_part(Tokens),
p_opt_nm2(Rest, opt_name_acc_new(NamePart)).
p_opt_nm2([?t('.') | Rest], Acc) ->
{NamePart, Rest2} = p_option_name_part(Rest),
Acc1 = opt_name_acc_add(NamePart, Acc),
p_opt_nm2(Rest2, Acc1);
p_opt_nm2(Tokens, Acc) ->
Res = opt_name_acc_finalize(Acc),
{Res, Tokens}.
opt_name_acc_new(Name) when is_atom(Name) ->
Name;
opt_name_acc_new(Name) when is_tuple(Name) ->
[Name].
opt_name_acc_add(Name, Acc) when is_atom(Name) ->
Acc1 = ensure_atom_list(Acc),
[Name | Acc1];
opt_name_acc_add(Name, Acc) when is_tuple(Name) ->
Acc1 = ensure_atom_list(Acc),
[Name | Acc1].
opt_name_acc_finalize(A) when is_atom(A) -> A;
opt_name_acc_finalize(L) when is_list(L) -> lists:reverse(L).
ensure_atom_list(A) when is_atom(A) -> [A];
ensure_atom_list(L) when is_list(L) -> L.
p_option_name_part(Tokens) ->
case Tokens of
[?t('(') | Rest] ->
case p_dotted_name(Rest) of
{DottedName, [?t(')') | Rest2]} ->
{list_to_tuple(undot_but_first(DottedName)), Rest2};
_ ->
?syntax_error(Rest, "expected option name")
end;
[?w(Ident/binary) | Rest] ->
{word_value(Ident), Rest};
_ ->
?syntax_error(Tokens, "expected option name")
end.
undot_but_first(['.' | Rest]) -> ['.' | undot2(Rest)];
undot_but_first(Components) -> undot2(Components).
undot2(Components) ->
lists:filter(fun(C) -> C /= '.' end, Components).
%% opt_list -> opt ',' opt_list
%% opt_list -> opt
%%
%% opt -> option_name '=' constant
%% opt -> option_name
p_opt_list(Tokens, Acc) ->
case Tokens of
[?t(']') | Rest] ->
{lists:reverse(Acc), Rest};
[?t(',') | Rest] ->
p_opt_list(Rest, Acc);
_ ->
{OptionName, Rest} = p_option_name(Tokens),
{Option, Rest4} =
case Rest of
[?t('=') | Rest2] ->
{Value, Rest3} = p_option_value(Rest2),
{{OptionName, Value}, Rest3};
_ ->
{OptionName, Rest}
end,
Acc1 = [Option | Acc],
p_opt_list(Rest4, Acc1)
end.
p_maybe_opt_list([?t('[') | Rest]) ->
{Opts, Rest2} = p_opt_list(Rest, []),
{Opts, Rest2};
p_maybe_opt_list(Tokens) ->
{[], Tokens}.
p_integer_const([?i(Int) | Rest]) -> {int_value(Int), Rest};
p_integer_const([?t('-'), ?i(Int) | Rest]) -> {-int_value(Int), Rest};
p_integer_const([?t('+'), ?i(Int) | Rest]) -> {int_value(Int), Rest};
p_integer_const(Tokens) ->
?syntax_error(Tokens).
%% const -> integer_or_float
%% const -> '+' integer_or_float
%% const -> '-' integer_or_float
%% const -> pstring_expr
%% const -> word
%%
%% integer_or_float -> float_lit
%% integer_or_float -> inf
%% integer_or_float -> nan
%% integer_or_float -> int_lit
p_const([?s(Str) | Rest]) ->
p_str_const(Rest, [str_value(Str)]);
p_const([?i(Int) | Rest]) ->
{int_value(Int), Rest};
p_const([?fl(Float) | Rest]) ->
{float_value(Float), Rest};
p_const([?w("inf") | Rest]) ->
{infinity, Rest};
p_const([?w("nan") | Rest]) ->
{nan, Rest};
p_const([?t('-'), ?i(Int) | Rest]) ->
{-int_value(Int), Rest};
p_const([?t('-'), ?fl(Float) | Rest]) ->
{-float_value(Float), Rest};
p_const([?t('-'), ?w("inf") | Rest]) ->
{'-infinity', Rest};
p_const([?t('+'), ?i(Int) | Rest]) ->
{int_value(Int), Rest};
p_const([?t('+'), ?fl(Float) | Rest]) ->
{float_value(Float), Rest};
p_const([?t('+'), ?w("inf") | Rest]) ->
{infinity, Rest};
p_const([?w(Word/binary) | Rest]) ->
{word_value(Word), Rest};
p_const(Tokens) ->
?syntax_error(Tokens, "expected constant").
p_str_const([?s(Str) | Rest], Acc) ->
Acc1 = [str_value(Str) | Acc],
p_str_const(Rest, Acc1);
p_str_const(Rest, Acc) ->
S = lists:concat(lists:reverse(Acc)),
{S, Rest}.
skip_semicolon([?t(';') | Rest]) -> skip_semicolon(Rest);
skip_semicolon(Tokens) -> Tokens.
verify_syntax("proto2") ->
ok;
verify_syntax("proto3") ->
ok;
verify_syntax("proto"++_ = Unsupported) ->
{error, "Unsupported proto version: " ++ Unsupported};
verify_syntax(Unsupported) ->
{error, ?f("Unsupported proto syntax: ~p", [Unsupported])}.
str_value(S) ->
S.
word_value(Bin) ->
list_to_atom(binary_to_list(Bin)).
int_value({dec, N}) -> N;
int_value({hex, N}) -> N;
int_value({oct, N}) -> N.
float_value(Float) ->
Float.
tokens_to_str(Tokens) ->
space_join([token_to_str(T) || T <- Tokens]).
token_to_str({_X, _Loc, Orig}) ->
if is_binary(Orig) ->
unicode:characters_to_list(Orig);
is_atom(Orig) ->
atom_to_list(Orig)
end.
space_join([]) -> "";
space_join([H | Tl]) -> [H | [[$\s, Elem] || Elem <- Tl]].
ensure_str(S) ->
case io_lib:printable_list(S) of
true ->
S;
false ->
?f("~p", [S])
end.
%% -- bwd compat -- do not use these ----------------------------------
%% Deprecated!
%% Instead, to retrieve proto definitions,
%% use the option to_proto_defs
%% with gpb_compile:file or gpb_compile:string
%% For exprotobuf, see also
%% https://github.com/bitwalker/exprotobuf/issues/114
%% Use the option to_proto_defs with gpb_compile:file/string instead.
post_process_one_file(FileName, Defs, Opts) ->
gpb_defs:post_process_one_file(FileName, Defs, Opts).
%% Use the option to_proto_defs with gpb_compile:file/string instead.
post_process_all_files(Defs, _Opts) ->
case gpb_defs:post_process_all_files(Defs, _Opts) of
{ok, Defs1} ->
gpb_defs:convert_defs_from_latest_version(Defs1, 1);
{error, Reason} ->
{error, Reason}
end.
%% Use the option to_proto_defs with gpb_compile:file/string instead.
format_post_process_error({error, Reasons}) ->
gpb_defs:format_post_process_error({error, Reasons}).
%% Use gpb_defs:fetch_imports instead.
fetch_imports(Defs) ->
gpb_defs:fetch_imports(Defs).
%% --^^--- bwd compat -- do not use these ----------------------------