Packages
gpb
4.9.2
5.0.0
4.21.7
4.21.6
4.21.5
4.21.4
4.21.3
4.21.2
4.21.1
4.21.0
4.20.0
4.19.9
4.19.8
4.19.7
4.19.6
4.19.5
4.19.4
4.19.3
4.19.2
4.19.1
4.19.0
4.18.0
4.17.7
4.17.6
4.17.5
4.17.3
4.17.2
4.17.1
4.17.0
4.16.2
4.16.1
4.16.0
4.15.2
4.15.1
4.14.2
4.14.1
4.14.0
4.13.0
4.12.0
4.11.2
4.11.1
4.11.0
4.10.6
4.10.5
4.10.4
4.10.3
4.10.2
4.10.1
4.10.0
4.9.3
4.9.2
4.9.1
4.9.0
4.8.0
4.7.3
4.7.2
4.7.1
4.7.0
4.6.0
4.5.1
4.5.0
4.4.1
4.4.0
4.3.3
4.3.2
4.3.1
4.3.0
4.2.3
4.2.2
4.2.1
4.2.0
4.1.9
4.1.8
4.1.7
4.1.6
4.1.5
4.1.4
4.1.3
4.1.2
4.1.1
4.1.0
4.0.2
4.0.1
4.0.0
3.28.1
3.28.0
3.27.7
3.27.6
3.27.5
3.27.4
3.27.3
3.27.2
3.27.1
3.27.0
3.26.8
3.26.7
3.26.6
3.26.5
3.26.4
3.26.3
3.26.2
3.26.1
3.26.0
3.25.2
3.25.1
3.25.0
3.24.4
3.24.3
3.24.2
3.24.1
3.24.0
3.23.2
3.23.1
3.23.0
3.22.5
3.22.4
3.22.3
3.22.2
3.22.1
3.22.0
3.21.3
3.21.2
3.21.1
3.21.0
3.20.3
3.20.2
3.20.0
3.19.0
3.18.10
3.18.9
3.18.8
3.18.7
3.18.6
3.18.5
3.18.4
3.18.3
3.18.2
3.18.1
3.18.0
3.17.13
3.17.12
3.17.11
3.17.10
3.17.9
3.17.8
3.17.5
3.17.4
3.17.3
3.17.2
3.17.1
3.17.0
3.16.0
3.15.0
3.14.0
3.13.0
3.12.2
3.12.1
3.12.0
3.11.0
A compiler for Google protocol buffer definitions files for Erlang.
Current section
Files
Jump to
Current section
Files
src/gpb_defs.erl
%%% Copyright (C) 2019 Tomas Abrahamsson
%%%
%%% Author: Tomas Abrahamsson <tab@lysator.liu.se>
%%%
%%% This library is free software; you can redistribute it and/or
%%% modify it under the terms of the GNU Lesser General Public
%%% License as published by the Free Software Foundation; either
%%% version 2.1 of the License, or (at your option) any later version.
%%%
%%% This library is distributed in the hope that it will be useful,
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
%%% Lesser General Public License for more details.
%%%
%%% You should have received a copy of the GNU Lesser General Public
%%% License along with this library; if not, write to the Free Software
%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
%%% MA 02110-1301 USA
%%% ------------------------------------------------------------------
%%% @doc Operations on definitions
%%% @end
%%% ------------------------------------------------------------------
-module(gpb_defs).
-export([post_process_one_file/3]).
-export([post_process_all_files/2]).
-export([format_post_process_error/1]).
-export([fetch_imports/1]).
-export_type([defs/0, def/0]).
-export_type([field/0]).
-include("../include/gpb.hrl").
-type defs() :: [def()].
-type def() :: {{msg, Name::atom()}, [field()]} |
{{group, Name::atom()}, [field()]} |
{{enum, Name::atom()}, [{Sym::atom(), Value::integer()}]} |
{{service, Name::atom()}, [#?gpb_rpc{}]} |
{package, Name::atom()} |
{syntax, string()} | % "proto2" | "proto3"
{{extensions, MsgName::atom()}, [field_number_extension()]} |
{{extend, MsgName::atom()}, MoreFields::[field()]} |
{proto3_msgs, [MsgName::atom()]} |
{{reserved_numbers, MsgName::atom()}, [integer()]} |
{{reserved_names, MsgName::atom()}, [FieldName::atom()]} |
{import, ProtoFile::string()} |
{{msg_options, MsgName::atom()}, [msg_option()]} |
{{msg_containment, ProtoName::string()}, [MsgName::atom()]} |
{{pkg_containment, ProtoName::string()}, PkgName::atom()} |
{{service_containment, ProtoName::string()},
[ServiceName::atom()]} |
{{rpc_containment, ProtoName::string()},
[{ServiceName::atom(), RpcName::atom()}]} |
{{enum_containment, ProtoName::string()}, [EnumName::atom()]} |
{file, {BaseSansExt::string(), Base::string()}}.
-type field() :: #?gpb_field{} | #gpb_oneof{}.
-type field_number_extension() :: {Lower::integer(), Upper::integer() | max}.
-type msg_option() :: {[NameComponent::atom()], OptionValue::term()}.
%% @hidden
%% @doc Post-process definitions from one file
post_process_one_file(FileName, Defs, Opts) ->
case find_package_def(Defs, Opts) of
{ok, Package} ->
Defs1 = handle_proto_syntax_version_one_file(
join_any_msg_options(
convert_default_values(
flatten_qualify_defnames(Defs, Package)))),
MetaInfo = mk_meta_info(FileName, Defs1, Opts),
{ok, [{file, {FileName, FileName}} | MetaInfo] ++ Defs1};
{error, Reasons} ->
{error, Reasons}
end.
%% @hidden
%% @doc Post-process definitions once the file and all its imports
%% each have been parsed and processed.
post_process_all_files(Defs, _Opts) ->
case resolve_names(Defs) of
{ok, Defs2} ->
{ok, normalize_msg_field_options(
handle_proto_syntax_version_all_files(
enumerate_msg_fields(
shorten_file_paths(
reformat_names(
extend_msgs(Defs2))))))};
{error, Reasons} ->
{error, Reasons}
end.
%% -> {ok, Defs} | {error, [Reason]}
resolve_names(Defs) ->
case resolve_refs(Defs) of
{ok, RDefs} ->
case verify_defs(RDefs) of
ok ->
{ok, RDefs};
{error, Reasons} ->
{error, Reasons}
end;
{error, Reasons} ->
{error, Reasons}
end.
shorten_file_paths(Defs) ->
Paths = [Path || {file, {Path, Path}} <- Defs],
PathsNoExts = [gpb_lib:drop_filename_ext(P) || P <- Paths],
Bases = try gpb_lib:basenameify_ish(PathsNoExts)
catch error:{gpb_error, {multiply_defined_file_or_files, _}} ->
gpb_lib:basenameify_ish(Paths)
end,
Mapping = lists:zip(Paths, Bases),
Defs2 = lists:map(
fun({file, {P, P}}) ->
{P, BaseishSansExt} = lists:keyfind(P, 1, Mapping),
Baseish = gpb_lib:copy_filename_ext(BaseishSansExt, P),
{file, {BaseishSansExt, Baseish}};
({file, _}=Def) ->
error({unexpected_file_def, Def});
(Other) ->
Other
end,
Defs),
shorten_meta_info(Mapping, Defs2).
%% Find any package specifier. At most one such package specifier
%% may exist, and it can exist anywhere (top-level) in the proto file,
%% yet it still applies to the whole file.
find_package_def(Defs, Opts) ->
case proplists:get_bool(use_packages, Opts) of
true ->
case [Pkg || {package, Pkg} <- Defs] of
[] ->
{ok, empty_pkg_root()};
[Pkg] ->
{ok, ['.' | Pkg]};
Pkgs when length(Pkgs) >= 2 ->
PrettyPkgs = [reformat_name(Pkg) || Pkg <- Pkgs],
{error, [{multiple_pkg_specifiers, PrettyPkgs}]}
end;
false ->
{ok, empty_pkg_root()}
end.
empty_pkg_root() ->
['.'].
%% For nested message definitions such as
%% ```
%% message m1 {
%% required uint32 f1 = 1;
%% message m2 { ... }
%% enum e2 { ... }
%% };",
%% '''
%% the parser will produce a nested structure, such as:
%% ```
%% [{{msg,M1},[#field{},
%% {{msg,M2}, [...]},
%% {{enum,E2}, [...]}]}]
%% '''
%% Flattening means to lift the nested m2 and e2 definition to the top-level,
%% so the above turns into:
%% ```
%% [{{msg,M1},[#field{}]},
%% {{msg,M2}, [...]},
%% {{enum,E2}, [...]}]
%% '''
%%
%% During this process, the message and enum names and similar get
%% fully qualified into absolute rooted name-paths. In the example
%% above, this applies to m1, m2 and e2. Note that at this stage,
%% nothing is done to resolve reference to names, such as message
%% types for fields. A name-path is a list of path components,
%% separated by the dot-atom, '.', and an absolute rooted name-path is
%% a path that begins with the dot-atom, '.', much like a slash or a
%% backslash in a file name path.
flatten_qualify_defnames(Defs, Root) ->
lists:reverse(
lists:foldl(
fun({{msg,Name}, FieldsOrDefs}, Acc) ->
FullName = prepend_path(Root, Name),
{Fields2, Defs2} = flatten_fields(FieldsOrDefs, FullName),
[{{msg,FullName},Fields2} | Defs2] ++ Acc;
({{group,FullName}, FieldsOrDefs}, Acc) ->
{Fields2, Defs2} = flatten_fields(FieldsOrDefs, FullName),
[{{group,FullName},Fields2} | Defs2] ++ Acc;
({{enum,Name}, ENs}, Acc) ->
FullName = prepend_path(Root, Name),
[{{enum,FullName}, ENs} | Acc];
({extensions,Exts}, Acc) ->
[{{extensions,Root},Exts} | Acc];
({{extend,{eref1,Name}}, FieldsOrDefs}, Acc) ->
FullNameCandidates =
rootward_names(Root, Name) ++
rootward_names(empty_pkg_root(), Name),
{Fields2, Defs2} = flatten_fields(FieldsOrDefs, Root),
[{{extend,{eref2,Root,FullNameCandidates}},Fields2} | Defs2] ++
Acc;
({{service, Name}, RPCs}, Acc) ->
FullName = prepend_path(Root, Name),
[{{service,FullName}, RPCs} | Acc];
(OtherElem, Acc) ->
[OtherElem | Acc]
end,
[],
Defs)).
flatten_fields(FieldsOrDefs, FullName) ->
{RFields2, Defs2} =
lists:foldl(
fun(#?gpb_field{}=F, {Fs,Ds}) ->
{[F | Fs], Ds};
(#gpb_oneof{}=O, {Fs,Ds}) ->
{[O | Fs], Ds};
({group1,TmpGName,GFields,MField}, {Fs,Ds}) ->
FullGroupName = prepend_path(FullName, TmpGName),
Group0 = {{group,FullGroupName}, GFields},
QDefs = flatten_qualify_defnames([Group0], FullGroupName),
MField1 = MField#?gpb_field{type={ref,FullGroupName}},
{[MField1 | Fs], QDefs++Ds};
({{extend, _Ref},_}=Def, {Fs,Ds}) ->
QDefs = flatten_qualify_defnames([Def], FullName),
{Fs, QDefs ++ Ds};
({reserved_numbers, Ns}, {Fs,Ds}) ->
Def = {{reserved_numbers,FullName}, Ns},
{Fs, [Def | Ds]};
({reserved_names, Ns}, {Fs,Ds}) ->
Def = {{reserved_names,FullName}, Ns},
{Fs, [Def | Ds]};
({option,OptName,OptValue}, {Fs,Ds}) ->
{Fs, [{{msg_option,FullName},{OptName,OptValue}} | Ds]};
(Def, {Fs,Ds}) ->
QDefs = flatten_qualify_defnames([Def], FullName),
{Fs, QDefs++Ds}
end,
{[],[]},
FieldsOrDefs),
{lists:reverse(RFields2), Defs2}.
%% Resolve any refs
resolve_refs(Defs) ->
Root = ['.'],
{ResolvedRefs, Reasons} =
lists:mapfoldl(
fun({{msg,FullName}, Fields}, Acc) ->
{NewFields, Acc2} =
resolve_field_refs(Fields, Defs, Root, FullName, Acc),
{{{msg,FullName}, NewFields}, Acc2};
({{group,FullName}, Fields}, Acc) ->
{NewFields, Acc2} =
resolve_field_refs(Fields, Defs, Root, FullName, Acc),
{{{group,FullName}, NewFields}, Acc2};
({{service,FullName}, Rpcs}, Acc) ->
{NewRPCs, Acc2} =
resolve_rpc_refs(Rpcs, Defs, Root, FullName, Acc),
{{{service,FullName}, NewRPCs}, Acc2};
({{extend,ExtendeeCandidates}, Fields}, Acc) ->
{Extendee, NewFields, Acc2} =
resolve_extend_refs(ExtendeeCandidates, Fields, Defs,
Root, Acc),
{{{extend,Extendee}, NewFields}, Acc2};
(OtherElem, Acc) ->
{OtherElem, Acc}
end,
[],
Defs),
if Reasons == [] -> {ok, ResolvedRefs};
Reasons /= [] -> {error, lists:reverse(Reasons)}
end.
resolve_field_refs(Fields, Defs, Root, FullName, Reasons) ->
lists:mapfoldl(
fun(#?gpb_field{name=FName, type={ref,Ref}}=Field, Acc) ->
case resolve_ref(Defs, Ref, Root, FullName) of
{found, TypeName} ->
{Field#?gpb_field{type=TypeName}, Acc};
not_found ->
Reason = {ref_to_undefined_msg_or_enum,
{{FullName, FName}, Ref}},
{Field, [Reason | Acc]}
end;
(#?gpb_field{name=FName, type={map,KeyType,{ref,Ref}}}=Field, Acc) ->
case resolve_ref(Defs, Ref, Root, FullName) of
{found, TypeName} ->
{Field#?gpb_field{type={map,KeyType,TypeName}}, Acc};
not_found ->
Reason = {ref_to_undefined_msg_or_enum,
{{FullName, FName}, Ref}},
{Field, [Reason | Acc]}
end;
(#?gpb_field{}=Field, Acc) ->
{Field, Acc};
(#gpb_oneof{fields=OFields1}=Oneof, Acc) ->
{OFields2, Acc2} =
resolve_field_refs(OFields1, Defs, Root, FullName, Acc),
{Oneof#gpb_oneof{fields=OFields2}, Acc2}
end,
Reasons,
Fields).
resolve_rpc_refs(Rpcs, Defs, Root, FullName, Reasons) ->
lists:mapfoldl(
fun({RpcName, {Arg, ArgIsStream}, {Return, ReturnIsStream}, Opts}=Rpc,
Acc) ->
case resolve_ref(Defs, Arg, Root, FullName) of
{found, {msg, MArg}} ->
case resolve_ref(Defs, Return, Root, FullName) of
{found, {msg, MReturn}} ->
NewOpts = [{reformat_name(Name), Value}
|| {option,Name,Value} <- Opts],
NewRpc = #?gpb_rpc{name=RpcName,
input=MArg,
input_stream=ArgIsStream,
output=MReturn,
output_stream=ReturnIsStream,
opts=NewOpts},
{NewRpc, Acc};
{found, {BadType, MReturn}} ->
Reason = {rpc_return_ref_to_non_msg,
{{FullName, RpcName, Return},
BadType, MReturn}},
{Rpc, [Reason | Acc]};
not_found ->
Reason = {rpc_return_ref_to_undefined_msg,
{{FullName, RpcName}, Return}},
{Rpc, [Reason | Acc]}
end;
{found, {BadType, MArg}} ->
Reason = {rpc_arg_ref_to_non_msg,
{{FullName, RpcName, Arg}, BadType, MArg}},
{Rpc, [Reason | Acc]};
not_found ->
Reason = {rpc_arg_ref_to_undefined_msg,
{{FullName, RpcName}, Arg}},
{Rpc, [Reason | Acc]}
end
end,
Reasons,
Rpcs).
resolve_extend_refs({eref2, Ctxt, ExtendeeCandidates}, Fields, Defs,
Root, Acc) ->
case resolve_ref_candidates(Defs, ExtendeeCandidates) of
{found, {msg,NewToBeExtended}} ->
{NewFields, Acc2} =
resolve_field_refs(Fields, Defs, Root, Ctxt, Acc),
{NewToBeExtended, NewFields, Acc2};
not_found ->
Reason = {extend_ref_to_undefined_msg, hd(ExtendeeCandidates)},
{hd(ExtendeeCandidates), Fields, [Reason | Acc]}
end.
%% -> {found, {msg,FullName}|{enum,FullName}} | not_found
resolve_ref(Defs, Ref, Root, FullName) ->
case is_absolute_ref(Ref) of
true ->
FullRef = ensure_path_prepended(Root, Ref),
find_typename(FullRef, Defs);
false ->
PossibleRoots = compute_roots(FullName),
find_ref_rootwards(PossibleRoots, Ref, Defs)
end.
resolve_ref_candidates(Defs, [Cand1 | Rest]) ->
case find_typename(Cand1, Defs) of
{found, TypeName} -> {found, TypeName};
not_found -> resolve_ref_candidates(Defs, Rest)
end;
resolve_ref_candidates(_Defs, []) ->
not_found.
find_ref_rootwards([PossibleRoot | Rest], Ref, Defs) ->
FullRef = ensure_path_prepended(PossibleRoot, Ref),
case find_typename(FullRef, Defs) of
{found, TypeName} -> {found, TypeName};
not_found -> find_ref_rootwards(Rest, Ref, Defs)
end;
find_ref_rootwards([], _Ref, _Defs) ->
not_found.
is_absolute_ref(['.' | _]) -> true;
is_absolute_ref(_Other) -> false.
find_typename(Name, [{{enum,Name}, _Values} | _]) -> {found, {enum,Name}};
find_typename(Name, [{{msg,Name}, _SubElems} | _]) -> {found, {msg,Name}};
find_typename(Name, [{{group,Name}, _Elems} | _]) -> {found, {group,Name}};
find_typename(Name, [_ | Rest]) -> find_typename(Name, Rest);
find_typename(_Name,[]) -> not_found.
%% Similar to compute_roots/1, but always keep `Name' last.
%% Example: rootward_names(['.',m1,'.',m2], x) ->
%% [['.',m1,'.',m2,'.',x],
%% ['.',m1,'.',x]
%% ['.',x]]
rootward_names(Path, Name) ->
[prepend_path(R, Name) || R <- compute_roots(Path)].
%% Turn ['.',m1,'.',m2,'.',m3]
%% into [['.',m1,'.',m2,'.',m3],
%% ['.',m1,'.',m2],
%% ['.',m1],
%% ['.']]
compute_roots(['.']) -> [['.']];
compute_roots(DeeperPath) ->
[DeeperPath | compute_roots(drop_last_level(DeeperPath))].
drop_last_level(['.']) -> ['.'];
drop_last_level(['.', X]) when is_atom(X) -> ['.'];
drop_last_level(DeeperPath) when length(DeeperPath) >= 3 ->
[_X, '.' | RestReversed] = lists:reverse(DeeperPath),
lists:reverse(RestReversed).
prepend_path(['.'], Id) when is_atom(Id) -> ['.', Id];
prepend_path(['.'], SubPath) when is_list(SubPath) -> ['.' | SubPath];
prepend_path(Path, Id) when is_atom(Id) -> Path ++ ['.', Id];
prepend_path(Path, SubPath) when is_list(SubPath) -> Path ++ ['.' | SubPath].
ensure_path_prepended(Pkg, Path) ->
case lists:prefix(Pkg, Path) of
false -> prepend_path(Pkg, Path);
true -> Path
end.
convert_default_values(Defs) ->
lists:map(
fun({{msg,Name},Fields}) ->
Fields2 = lists:map(fun convert_default_values_field/1, Fields),
{{msg,Name},Fields2};
({{group,Name},Fields}) ->
Fields2 = lists:map(fun convert_default_values_field/1, Fields),
{{group,Name},Fields2};
(Other) ->
Other
end,
Defs).
convert_default_values_field(#?gpb_field{type=Type, opts=Opts}=Field) ->
case {Type, lists:keyfind(default, 1, Opts)} of
{bytes, {default, Default}} when is_list(Default) ->
%% Default values for type bytes are written as a string
Default2 = list_to_binary(Default),
Opts2 = lists:keyreplace(default, 1, Opts, {default, Default2}),
Field#?gpb_field{opts=Opts2};
_ ->
Field
end;
convert_default_values_field(#gpb_oneof{fields=OFs}=Field) ->
OFs2 = lists:map(fun convert_default_values_field/1, OFs),
Field#gpb_oneof{fields=OFs2}.
join_any_msg_options(Defs) ->
{NonMsgOptDefs, MsgOptsDict} =
lists:foldl(
fun({{msg_option,MsgName},Opt}, {Ds,MsgOptsDict}) ->
{Ds, dict:append(MsgName, Opt, MsgOptsDict)};
(OtherDef, {Ds, MsgOptsDict}) ->
{[OtherDef | Ds], MsgOptsDict}
end,
{[], dict:new()},
Defs),
MsgOpts = [{{msg_options, MsgName}, MsgOpts}
|| {MsgName, MsgOpts} <- dict:to_list(MsgOptsDict)],
lists:reverse(NonMsgOptDefs, MsgOpts).
handle_proto_syntax_version_one_file(Defs) ->
case proplists:get_value(syntax, Defs) of
undefined -> handle_proto2_1(Defs);
"proto2" -> handle_proto2_1(Defs);
"proto3" -> handle_proto3_1(Defs)
end.
handle_proto2_1(Defs) ->
Defs.
handle_proto3_1(Defs) ->
%% FIXME: Verify no 'extensions' or 'extend'
%% FIXME: Verify no 'required' occurrences
%% FIXME: Verify enums start with 0
%% Remember which msgs were defined using proto3 syntax,
%% so we can treat them differently later on.
anno_msgs_proto3_origin(Defs).
anno_msgs_proto3_origin(Defs) ->
anno_msgs_proto3_origin_2(Defs, []).
anno_msgs_proto3_origin_2([{{msg,Msg},_Fields}=Def | Rest], P3Msgs) ->
[Def | anno_msgs_proto3_origin_2(Rest, [Msg | P3Msgs])];
anno_msgs_proto3_origin_2([Def | Rest], Acc) ->
[Def | anno_msgs_proto3_origin_2(Rest, Acc)];
anno_msgs_proto3_origin_2([], Acc) ->
[{proto3_msgs,lists:reverse(Acc)}].
handle_proto_syntax_version_all_files(Defs) ->
P3Items = [X || {proto3_msgs,_}=X <- Defs],
if P3Items == [] ->
Defs;
P3Items /= [] ->
Proto3Msgs = lists:append([Msgs || {proto3_msgs,Msgs} <- P3Items]),
Defs1 = Defs -- P3Items,
Defs2 = Defs1 ++ [{proto3_msgs, lists:sort(Proto3Msgs)}],
%% The protobuf language guide for proto3 says: "In proto3,
%% repeated fields of scalar numeric types use packed encoding by
%% default."
default_repeated_to_packed(Defs2, Proto3Msgs)
end.
default_repeated_to_packed(Defs, P3Msgs) ->
lists:map(
fun({{msg,MsgName},Fields}=MsgDef) ->
case lists:member(MsgName, P3Msgs) of
true ->
Fields1 = default_repeated_fields_to_packed(Fields),
{{msg,MsgName}, Fields1};
false ->
MsgDef
end;
(Other) ->
Other
end,
Defs).
default_repeated_fields_to_packed(Fields) ->
lists:map(
fun(#?gpb_field{occurrence=repeated, opts=Opts, type=Type}=F) ->
case {proplists:get_value(packed, Opts),
is_scalar_numeric(Type)} of
{undefined, true} ->
NewOpts = [{packed, true} | Opts],
F#?gpb_field{opts=NewOpts};
_ ->
F
end;
(F) ->
F
end,
Fields).
is_scalar_numeric(int32) -> true;
is_scalar_numeric(int64) -> true;
is_scalar_numeric(uint32) -> true;
is_scalar_numeric(uint64) -> true;
is_scalar_numeric(sint32) -> true;
is_scalar_numeric(sint64) -> true;
is_scalar_numeric(fixed32) -> true;
is_scalar_numeric(fixed64) -> true;
is_scalar_numeric(sfixed32) -> true;
is_scalar_numeric(sfixed64) -> true;
is_scalar_numeric(bool) -> true;
is_scalar_numeric(float) -> true;
is_scalar_numeric(double) -> true;
is_scalar_numeric({enum,_}) -> true;
is_scalar_numeric(_) -> false. % not: string | bytes | msg | map
%% Find inconsistencies
%%
%% Prerequisites:
%% `Defs' is expected to be flattened and may or may not be reformatted.
verify_defs(Defs) ->
collect_errors(Defs,
[{msg, [fun verify_field_defaults/2]},
{group, [fun verify_field_defaults/2]},
{extend, [fun verify_extend/2]},
{service, [fun verify_service/2]},
{'_', [fun(_Def, _AllDefs) -> ok end]}]).
collect_errors(Defs, VerifiersList) ->
collect_errors(Defs, Defs, VerifiersList, ok).
collect_errors([{{ElemType,_},_}=Def | Rest], AllDefs, VerifiersList, Acc) ->
Result = lists:foldl(
fun(Verifier, A) -> add_acc(A, Verifier(Def, AllDefs)) end,
Acc,
find_verifiers(ElemType, VerifiersList)),
collect_errors(Rest, AllDefs, VerifiersList, Result);
collect_errors([_OtherDef | Rest], AllDefs, VerifiersList, Acc) ->
%% Example: import, package, ...
collect_errors(Rest, AllDefs, VerifiersList, Acc);
collect_errors([], _AllRefs, _VerifiersList, Acc) ->
case Acc of
ok -> ok;
{error, ReasonsReversed} -> {error, lists:reverse(ReasonsReversed)}
end.
add_acc(AnyPreviousResult, ok) -> AnyPreviousResult;
add_acc(ok, {error, R}) -> {error, add_reason([], R)};
add_acc({error, Reasons}, {error, R}) -> {error, add_reason(Reasons, R)}.
add_reason(Reasons, Reason) when not is_list(Reason) ->
[Reason | Reasons];
add_reason(Reasons, MoreReasons) when is_list(MoreReasons) ->
lists:reverse(MoreReasons, Reasons).
find_verifiers(Type, [{Type, Verifiers} | _]) -> Verifiers;
find_verifiers(_Type, [{'_', Verifiers} | _]) -> Verifiers;
find_verifiers(Type, [_Other | Rest]) -> find_verifiers(Type, Rest).
verify_field_defaults({{msg,M}, Fields}, AllDefs) ->
lists:foldl(fun(#?gpb_field{name=Name, type=Type, opts=FOpts}, Acc) ->
Res = case lists:keysearch(default, 1, FOpts) of
{value, {default, Default}} ->
verify_scalar_default_if_present(
M, Name, Type, Default, AllDefs);
false ->
ok
end,
add_acc(Acc, Res);
(#gpb_oneof{fields=OFields}, Acc) ->
Res = verify_field_defaults({{msg,M},OFields}, AllDefs),
add_acc(Acc, Res)
end,
ok,
Fields);
verify_field_defaults({{group,G}, Fields}, AllDefs) ->
verify_field_defaults({{msg,G}, Fields}, AllDefs).
verify_scalar_default_if_present(MsgName, FieldName, Type, Default, AllDefs) ->
case Type of
{enum,Ref} ->
case lists:keysearch({enum, Ref}, 1, AllDefs) of
{value, {{enum,Ref}, Enumerators}} ->
case lists:keysearch(Default, 1, Enumerators) of
{value, {Default, _Value}} ->
ok;
false ->
{error,
{{invalid_default_enum_value, Default},
{name_to_dstr(MsgName), atom_to_list(FieldName)}}}
end;
false ->
ok %% caught by another verification step
end;
ScalarType when is_atom(ScalarType) ->
case gpb:check_scalar(Default, ScalarType) of
ok ->
ok;
{error, Reason} ->
{error, {Reason, {name_to_dstr(MsgName),
atom_to_list(FieldName)}}}
end
end.
verify_extend(_, _AllDefs) ->
%% FIXME
ok.
verify_service(_, _AllDefs) ->
%% FIXME
ok.
name_to_absdstr(['.' | Name]) -> "." ++ name_to_dstr(Name);
name_to_absdstr(Name) -> name_to_dstr(Name).
name_to_dstr(Name) when is_list(Name) ->
gpb_lib:dot_join([atom_to_list(P) || P <- Name, P /= '.']);
name_to_dstr(Name) when is_atom(Name) ->
atom_to_list(Name).
%% @hidden
%% Format error and reasons from the post processing stage.
format_post_process_error({error, Reasons}) ->
lists:flatten([[fmt_err(Reason),"\n"] || Reason <- Reasons]).
-define(f(F, A), io_lib:format(F, A)).
fmt_err({multiple_pkg_specifiers, Pkgs}) ->
?f("package specified more than once: ~s~n",
[gpb_lib:comma_join([atom_to_list(Pkg) || Pkg <- Pkgs])]);
fmt_err({ref_to_undefined_msg_or_enum, {{Msg, Field}, To}}) ->
?f("in msg ~s, field ~s: undefined reference ~s",
[name_to_dstr(Msg), name_to_dstr(Field), name_to_absdstr(To)]);
fmt_err({extend_ref_to_undefined_msg, Msg}) ->
?f("extend of unknown message ~s", [name_to_absdstr(Msg)]);
fmt_err({rpc_return_ref_to_non_msg,
{{FullName, RpcName, Return}, BadType, MReturn}}) ->
?f("in service ~s, rpc ~s, the return type, ~s, refers to "
" a ~p, ~s, instead of to a message",
[name_to_dstr(FullName), name_to_dstr(RpcName), name_to_absdstr(Return),
BadType, name_to_dstr(MReturn)]);
fmt_err({rpc_return_ref_to_undefined_msg, {{FullName, RpcName}, Ret}}) ->
?f("in service ~s, rpc ~s, return: undefined reference ~s",
[name_to_dstr(FullName), name_to_dstr(RpcName), name_to_absdstr(Ret)]);
fmt_err({rpc_arg_ref_to_non_msg, {{FullName, RpcName, Arg}, BadType, MArg}}) ->
?f("in service ~s, rpc ~s, the arg type, ~s, refers to "
" a ~p, ~s, instead of to a message",
[name_to_dstr(FullName), name_to_dstr(RpcName), name_to_absdstr(Arg),
BadType, name_to_dstr(MArg)]);
fmt_err({rpc_arg_ref_to_undefined_msg, {{FullName, RpcName}, Arg}}) ->
?f("in service ~s, rpc ~s, arg: undefined reference ~s",
[name_to_dstr(FullName), name_to_dstr(RpcName), name_to_absdstr(Arg)]);
fmt_err({{invalid_default_enum_value, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: undefined enumerator in default value ~s",
[Msg, Field, Default]);
fmt_err({{{value_out_of_range, Signedness, Bits}, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: default value ~p out of range for ~p ~p bit int",
[Msg, Field, Default, Signedness, Bits]);
fmt_err({{{bad_integer_value, Signedness, Bits}, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: bad default value ~p for ~p ~p bit int",
[Msg, Field, Default, Signedness, Bits]);
fmt_err({{bad_floating_point_value, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: bad floating point default value ~p",
[Msg, Field, Default]);
fmt_err({{bad_boolean_value, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: bad default value ~p for boolean",
[Msg, Field, Default]);
fmt_err({{bad_unicode_string, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: bad default value ~p for string",
[Msg, Field, Default]);
fmt_err({{bad_binary_value, Default}, {Msg, Field}}) ->
?f("in msg ~s, field ~s: bad default value ~p for bytes",
[Msg, Field, Default]).
%% Rewrites for instance ['.','m1','.',m2] into 'm1.m2'
%% Example: {{msg,['.','m1','.',m2]}, [#field{type={msg,['.','m1','.',m3]}}]}
%% becomes: {{msg,'m1.m2'}, [#field{type={msg,'m1.m3'}}]}
%%
%% Prerequisites:
%% `Defs' is expected to be flattened and names and references
%% are expected to have been resolved
reformat_names(Defs) ->
lists:map(fun({{msg,Name}, Fields}) ->
{{msg,reformat_name(Name)}, reformat_fields(Fields)};
({{group,Name}, Fields}) ->
{{group,reformat_name(Name)}, reformat_fields(Fields)};
({{msg_containment, ProtoName}, Msgs}) ->
{{msg_containment,ProtoName},
[reformat_name(N) || N <- Msgs]};
({{enum,Name}, ENs}) ->
{{enum,reformat_name(Name)}, reformat_enum_opt_names(ENs)};
({{enum_containment, ProtoName}, EnumNames}) ->
{{enum_containment,ProtoName},
[reformat_name(EnumName) || EnumName <- EnumNames]};
({{extensions,Name}, Exts}) ->
{{extensions,reformat_name(Name)}, Exts};
({{extend,Name}, Fields}) ->
{{extend,reformat_name(Name)}, reformat_fields(Fields)};
({{service,Name}, RPCs}) ->
{{service,reformat_name(Name)}, reformat_rpcs(RPCs)};
({{service_containment, ProtoName}, ServiceNames}) ->
{{service_containment,ProtoName},
[reformat_name(Name) || Name <- ServiceNames]};
({{rpc_containment, ProtoName}, RpcNames}) ->
{{rpc_containment,ProtoName},
[{reformat_name(ServiceName), RpcName}
|| {ServiceName,RpcName} <- RpcNames]};
({package, Name}) ->
{package, reformat_name(Name)};
({{pkg_containment, ProtoName}, PkgName}) ->
{{pkg_containment,ProtoName}, reformat_name(PkgName)};
({proto3_msgs,Names}) ->
{proto3_msgs,[reformat_name(Name) || Name <- Names]};
({{reserved_numbers,Name}, Ns}) ->
{{reserved_numbers,reformat_name(Name)}, Ns};
({{reserved_names,Name}, FieldNames}) ->
{{reserved_names,reformat_name(Name)}, FieldNames};
({{msg_options,MsgName}, Opt}) ->
{{msg_options,reformat_name(MsgName)}, Opt};
(OtherElem) ->
OtherElem
end,
Defs).
reformat_fields(Fields) ->
lists:map(
fun(#?gpb_field{type={T,Nm}}=F) ->
F#?gpb_field{type={T,reformat_name(Nm)}};
(#?gpb_field{type={map,KeyType,{T,Nm}}}=F) ->
F#?gpb_field{type={map,KeyType,{T,reformat_name(Nm)}}};
(#?gpb_field{}=F) ->
F;
(#gpb_oneof{fields=Fs}=O) ->
O#gpb_oneof{fields=reformat_fields(Fs)}
end,
Fields).
%% `Defs' is expected to be parsed.
reformat_enum_opt_names(Def) ->
[case Item of
{option, Name, Value} ->
{option, reformat_name(Name), Value};
Other ->
Other
end
|| Item <- Def].
reformat_name(Name) when is_atom(Name) -> Name;
reformat_name(Name) when is_list(Name) -> % dotted name components:
list_to_atom(gpb_lib:dot_join([atom_to_list(P) || P <- Name, P /= '.'])).
reformat_rpcs(RPCs) ->
lists:map(fun(#?gpb_rpc{name=RpcName, input=Arg, output=Return}=R) ->
R#?gpb_rpc{name=RpcName,
input=reformat_name(Arg),
output=reformat_name(Return)}
end,
RPCs).
%% `Defs' is expected to be flattened and may or may not be reformatted
%% `Defs' is expected to be verified, to not extend missing messages
extend_msgs(Defs0) ->
Extendings = [E || {{extend,_MsgToExtend},_MoreFields}=E <- Defs0],
lists:foldl(fun possibly_extend_msg/2, Defs0, Extendings).
possibly_extend_msg({{extend,Msg}, MoreFields}=Extending, Defs) ->
case lists:keyfind({msg,Msg}, 1, Defs) of
{{msg,Msg}, OrigFields} ->
NewDef = {{msg,Msg}, OrigFields ++ MoreFields},
lists:keyreplace({msg,Msg}, 1, Defs, NewDef) -- [Extending];
false ->
Defs
end.
%% `Defs' is expected to be flattened
enumerate_msg_fields(Defs) ->
lists:map(fun({{msg,Name}, Fields}) ->
{{msg, Name}, enumerate_fields(Fields)};
({{group,Name}, Fields}) ->
{{group, Name}, enumerate_fields(Fields)};
(OtherElem) ->
OtherElem
end,
Defs).
enumerate_fields(Fields) ->
lists:map(fun({I, #?gpb_field{}=F}) ->
F#?gpb_field{rnum=I};
({I, #gpb_oneof{fields=Fs}=O}) ->
NewFields = [F#?gpb_field{rnum=I} || F <- Fs],
O#gpb_oneof{rnum=I, fields=NewFields}
end,
index_seq(2, Fields)).
index_seq(_Start, []) -> [];
index_seq(Start, L) -> lists:zip(lists:seq(Start, length(L) + Start - 1), L).
%% `Defs' is expected to be parsed.
normalize_msg_field_options(Defs) ->
lists:map(fun({{msg,Name}, Fields}) ->
{{msg, Name}, normalize_field_options(Fields)};
({{group,Name}, Fields}) ->
{{group, Name}, normalize_field_options(Fields)};
(OtherElem) ->
OtherElem
end,
Defs).
normalize_field_options(Fields) ->
lists:map(fun(#?gpb_field{type={map,_KeyType,_ValueType}, opts=Opts}=F) ->
Opts1 = normalize_field_options_2(Opts),
Opts2 = Opts1 -- [packed],
F#?gpb_field{opts = Opts2};
(#?gpb_field{opts=Opts}=F) ->
Opts1 = normalize_field_options_2(Opts),
F#?gpb_field{opts = Opts1};
(#gpb_oneof{fields=Fs}=O) ->
O#gpb_oneof{fields=normalize_field_options(Fs)}
end,
Fields).
normalize_field_options_2(Opts) ->
Opts1 = opt_tuple_to_atom_if_defined_true(packed, Opts),
opt_tuple_to_atom_if_defined_true(deprecated, Opts1).
opt_tuple_to_atom_if_defined_true(Opt, Opts) ->
case proplists:get_bool(Opt, Opts) of
false -> lists:keydelete(Opt, 1, Opts);
true -> [Opt | lists:keydelete(Opt, 1, Opts)]
end.
%% @hidden
%% @doc Fetch the `import'ed files.
%% `Defs' is expected to be parsed, but not necessarily post_processed.
-spec fetch_imports(defs()) -> [ProtoFile::string()].
fetch_imports(Defs) ->
[Path || {import,Path} <- Defs].
mk_meta_info(FileName, Defs, Opts) ->
meta_msg_containment(FileName, Defs)
++ meta_enum_containment(FileName, Defs)
++ meta_pkg_containment(FileName, Defs, Opts)
++ meta_service_and_rpc_containment(FileName, Defs).
meta_msg_containment(FileName, Defs) ->
[{{msg_containment, FileName}, lists:sort(gpb_lib:msg_names(Defs))}].
meta_enum_containment(FileName, Defs) ->
[{{enum_containment, FileName}, lists:sort(gpb_lib:enum_names(Defs))}].
meta_pkg_containment(FileName, Defs, Opts) ->
case proplists:get_value(package, Defs, '$undefined') of
'$undefined' ->
[];
Pkg ->
case proplists:get_bool(use_packages, Opts) of
false ->
[];
true ->
[{{pkg_containment,FileName}, Pkg}]
end
end.
meta_service_and_rpc_containment(FileName, Defs) ->
Services = [{Name,RPCs} || {{service,Name}, RPCs} <- Defs],
if Services == [] ->
[];
true ->
ServiceNames = [Name || {Name, _RPCs} <- Services],
RpcNames = lists:append([[{SName, RName}
|| {RName, _In,_Out, _Opts} <- RPCs]
|| {SName, RPCs} <- Services]),
[{{service_containment, FileName}, lists:sort(ServiceNames)},
{{rpc_containment, FileName}, RpcNames}]
end.
shorten_meta_info(Mapping, Defs) ->
lists:map(
fun({{msg_containment, FileName}, MsgNames}) ->
{FileName, Baseish} = lists:keyfind(FileName, 1, Mapping),
{{msg_containment, Baseish}, MsgNames};
({{enum_containment, FileName}, EnumNames}) ->
{FileName, Baseish} = lists:keyfind(FileName, 1, Mapping),
{{enum_containment, Baseish}, EnumNames};
({{service_containment, FileName}, Services}) ->
{FileName, Baseish} = lists:keyfind(FileName, 1, Mapping),
{{service_containment, Baseish}, Services};
({{rpc_containment, FileName}, Rpcs}) ->
{FileName, Baseish} = lists:keyfind(FileName, 1, Mapping),
{{rpc_containment, Baseish}, Rpcs};
({{pkg_containment, FileName}, PkgName}) ->
{FileName, Baseish} = lists:keyfind(FileName, 1, Mapping),
{{pkg_containment, Baseish}, PkgName};
(Other) ->
Other
end,
Defs).