Packages
protox
1.6.9
2.0.9
2.0.8
2.0.7
2.0.6
2.0.5
2.0.4
2.0.3
2.0.2
2.0.1
2.0.0
2.0.0-dev
1.7.8
1.7.7
1.7.6
1.7.5
1.7.4
retired
1.7.3
1.7.2
1.7.1
1.7.0
1.6.10
1.6.9
1.6.8
1.6.7
1.6.6
1.6.5
1.6.4
1.6.3
1.6.2
1.6.1
1.6.0
1.5.1
1.5.0
1.4.0
1.3.2
1.3.1
1.3.0
1.2.4
1.2.3
1.2.2
1.2.1
1.2.0
1.1.1
1.1.0
1.0.0
0.25.0
0.24.0
0.23.1
0.23.0
0.22.0
0.21.0
0.20.0
0.19.1
0.19.0
0.18.0
0.17.0
0.16.2
0.16.1
0.16.0
0.15.2
0.15.1
0.15.0
0.14.0
0.13.0
0.12.1
0.12.0
0.11.1
0.11.0
0.10.0
0.9.1
0.9.0
0.8.0
0.7.1
0.7.0
A fast, easy to use and 100% conformant Elixir library for Google Protocol Buffers (aka protobuf)
Current section
Files
Jump to
Current section
Files
lib/protox/parse.ex
defmodule Protox.Parse do
@moduledoc false
# Internal.
# Creates definitions from a protobuf encoded description (Protox.Google.Protobuf.FileDescriptorSet)
# of a set of .proto files. This description is produced by `protoc`.
alias Protox.Field
alias Protox.Google.Protobuf.{
DescriptorProto,
FieldDescriptorProto,
FieldOptions,
FileDescriptorSet,
MessageOptions
}
@spec parse(binary, Keyword.t()) :: map()
def parse(file_descriptor_set, opts \\ []) do
{:ok, descriptor} = FileDescriptorSet.decode(file_descriptor_set)
namespace_or_nil = Keyword.get(opts, :namespace)
%{enums: %{}, messages: %{}}
|> parse_files(descriptor.file)
|> post_process(namespace_or_nil)
|> remove_well_known_types()
end
# -- Private
# As not all protoc installations come with the well-known types (Any, Duration, etc.),
# protox provides these types automatically.
# However, as user code can include those well-known types, we have to get rid of them
# here to make sure they are now defined more than once.
defp remove_well_known_types(acc) do
filtered_messages =
Enum.reject(acc.messages, fn {message_name, _syntax, _fields} ->
message_name in Google.Protobuf.well_known_types()
end)
filtered_enums =
Enum.reject(acc.enums, fn {enum_name, _constants} ->
enum_name in Google.Protobuf.well_known_types()
end)
%{enums: filtered_enums, messages: filtered_messages}
end
# canonization: camelization, fqdn, prepend with namespace
defp post_process(acc, namespace_or_nil) do
processed_messages =
for {mname, {syntax, fields}} <- acc.messages, into: [] do
{
Module.concat([namespace_or_nil | Enum.map(mname, &Macro.camelize(&1))]),
syntax,
Enum.map(
fields,
fn %Field{} = field ->
field
|> resolve_types(acc.enums)
|> set_default_value(acc.enums)
|> concat_names(namespace_or_nil)
end
)
}
end
processsed_enums =
for {ename, constants} <- acc.enums, into: [] do
{
Module.concat([namespace_or_nil | Enum.map(ename, &Macro.camelize(&1))]),
constants
}
end
%{enums: processsed_enums, messages: processed_messages}
end
defp resolve_types(%Field{type: {:to_resolve, tname}} = field, enums) do
if Map.has_key?(enums, tname) do
%Field{field | type: {:enum, tname}}
else
%Field{field | type: {:message, tname}}
end
end
defp resolve_types(%Field{kind: :map, type: {key_type, {:to_resolve, tname}}} = field, enums) do
if Map.has_key?(enums, tname) do
%Field{field | type: {key_type, {:enum, tname}}}
else
%Field{field | type: {key_type, {:message, tname}}}
end
end
defp resolve_types(%Field{} = field, _enums), do: field
defp set_default_value(
%Field{kind: {:scalar, :default_to_resolve}, type: {:enum, ename}} = field,
enums
) do
# proto2: the first entry is always the default value
# proto3: the entry with value 0 is the default value, and protoc mandates the first entry
# to have the value 0
[{_, first_is_default} | _] = Map.fetch!(enums, ename)
%Field{field | kind: {:scalar, first_is_default}, type: {:enum, ename}}
end
defp set_default_value(%Field{} = field, _enums), do: field
defp concat_names(%Field{type: {:enum, ename}} = field, namespace_or_nil) do
%Field{field | type: {:enum, Module.concat([namespace_or_nil | ename])}}
end
defp concat_names(%Field{type: {:message, mname}} = field, namespace_or_nil) do
%Field{field | type: {:message, Module.concat([namespace_or_nil | mname])}}
end
defp concat_names(
%Field{kind: :map, type: {key_type, {:message, mname}}} = field,
namespace_or_nil
) do
%Field{field | type: {key_type, {:message, Module.concat([namespace_or_nil | mname])}}}
end
defp concat_names(%Field{type: {key_type, {:enum, ename}}} = field, namespace_or_nil) do
%Field{field | type: {key_type, {:enum, Module.concat([namespace_or_nil | ename])}}}
end
defp concat_names(%Field{} = field, _), do: field
defp parse_files(acc, descriptors) do
Enum.reduce(descriptors, acc, fn descriptor, acc ->
parse_file(acc, descriptor)
end)
end
defp parse_file(acc, descriptor) do
syntax =
case descriptor.syntax do
"proto3" -> :proto3
"proto2" -> :proto2
"" -> :proto2
end
prefix =
case descriptor.package do
"" -> []
p -> p |> String.split(".") |> Enum.map(&Macro.camelize(&1))
end
acc
|> make_enums(prefix, descriptor.enum_type)
|> make_messages(syntax, prefix, descriptor.message_type)
|> add_extensions(nil, syntax, descriptor.extension)
end
defp make_enums(acc, prefix, descriptors) do
Enum.reduce(descriptors, acc, fn descriptor, acc ->
make_enum(acc, prefix, descriptor)
end)
end
defp make_enum(acc, prefix, descriptor) do
enum_name = prefix ++ [descriptor.name]
enum_constants = [] |> make_enum_constants(descriptor.value) |> Enum.reverse()
%{acc | enums: Map.put(acc.enums, enum_name, enum_constants)}
end
defp make_enum_constants(acc, []), do: acc
defp make_enum_constants(acc, [descriptor | descriptors]) do
make_enum_constants(
[{descriptor.number, String.to_atom(descriptor.name)} | acc],
descriptors
)
end
defp make_messages(acc, syntax, prefix, descriptors) do
Enum.reduce(descriptors, acc, fn descriptor, acc ->
make_message(acc, syntax, prefix, descriptor)
end)
end
defp make_message(acc, _syntax, _prefix, %DescriptorProto{
options: %MessageOptions{map_entry: map_entry}
})
when map_entry do
# This case has already been handled in the upper message with add_maps.
acc
end
defp make_message(acc, syntax, prefix, descriptor) do
name = prefix ++ [descriptor.name]
acc
|> add_message(syntax, name)
|> make_messages(syntax, name, descriptor.nested_type)
|> make_enums(name, descriptor.enum_type)
|> add_fields(descriptor, name, syntax, descriptor.field)
|> add_fields(descriptor, name, syntax, descriptor.extension)
end
defp add_message(acc, syntax, name) do
%{acc | messages: Map.put_new(acc.messages, name, {syntax, []})}
end
defp add_extensions(acc, upper, syntax, fields) do
Enum.reduce(fields, acc, fn field, acc ->
add_field(acc, syntax, upper, fully_qualified_name(field.extendee), field)
end)
end
defp add_fields(acc, upper, msg_name, syntax, fields) do
Enum.reduce(fields, acc, fn field, acc ->
add_field(acc, syntax, upper, msg_name, field)
end)
end
defp add_field(acc, syntax, upper, msg_name, descriptor) do
{label, kind, type} =
case map_entry(upper, msg_name, descriptor) do
nil ->
type = get_type(descriptor)
kind = get_kind(syntax, upper, descriptor)
{field_label(descriptor), kind, type}
map_type ->
{nil, :map, map_type}
end
field =
Field.new!(
tag: descriptor.number,
label: label,
name: String.to_atom(descriptor.name),
kind: kind,
type: type
)
new_messages =
Map.update!(acc.messages, msg_name, fn {syntax, fields} ->
{syntax, [field | fields]}
end)
%{acc | messages: new_messages}
end
defp field_label(%{proto3_optional: true}), do: :proto3_optional
defp field_label(%{label: label}), do: label
defp map_entry(nil, _prefix, _descriptor), do: nil
defp map_entry(
upper,
prefix,
%FieldDescriptorProto{label: :repeated, type: :message} = descriptor
) do
# Might be a map. Now find a nested type of upper that is the corresponding entry.
res =
Enum.find(upper.nested_type, fn m ->
if m.options != nil and m.options.map_entry do
m_name = prefix ++ [m.name]
t_name = fully_qualified_name(descriptor.type_name)
# Test if the generated name of the MapEntry message is the same as the one
# referenced by the actual map field.
m_name == t_name
else
false
end
end)
case res do
nil ->
nil
m ->
key_type = Enum.find(m.field, &(&1.name == "key")).type
value_type_field = Enum.find(m.field, &(&1.name == "value"))
value_type = get_type(value_type_field)
{key_type, value_type}
end
end
defp map_entry(_upper, _prefix, _descriptor), do: nil
defp fully_qualified_name(name) do
# Make sure first element is always ".".
true = String.starts_with?(name, ".")
name
|> String.split(".")
|> tl()
|> Enum.map(&Macro.camelize(&1))
end
import Protox.Guards
defp get_kind(_syntax, upper, %FieldDescriptorProto{oneof_index: index}) when index != nil do
parent = String.to_atom(Enum.at(upper.oneof_decl, index).name)
{:oneof, parent}
end
defp get_kind(_syntax, _upper, %FieldDescriptorProto{
label: :repeated,
options: %FieldOptions{packed: true}
}) do
:packed
end
defp get_kind(_syntax, _upper, %FieldDescriptorProto{
label: :repeated,
options: %FieldOptions{packed: false}
}) do
:unpacked
end
defp get_kind(:proto3, _upper, %FieldDescriptorProto{label: :repeated, type: :enum}) do
:packed
end
defp get_kind(:proto3, _upper, %FieldDescriptorProto{label: :repeated, type: ty})
when is_primitive(ty) do
:packed
end
defp get_kind(_syntax, _upper, %FieldDescriptorProto{label: :repeated}), do: :unpacked
defp get_kind(_syntax, _upper, %FieldDescriptorProto{label: label} = descriptor)
when label == :optional or label == :required do
{:scalar, get_default_value(descriptor)}
end
defp get_type(%FieldDescriptorProto{type_name: tyname}) when tyname != nil do
# Documentation in descriptor.proto says that it's possible that `type_name` is set, but not
# `type`. The type will be resolved in a post-process pass.
{:to_resolve, fully_qualified_name(tyname)}
end
defp get_type(descriptor), do: descriptor.type
defp get_default_value(%FieldDescriptorProto{type: :enum, default_value: nil}) do
:default_to_resolve
end
defp get_default_value(%FieldDescriptorProto{type: :enum} = f) do
String.to_atom(f.default_value)
end
defp get_default_value(%FieldDescriptorProto{type: :message}), do: nil
defp get_default_value(%FieldDescriptorProto{type: ty, default_value: nil}) do
Protox.Default.default(ty)
end
defp get_default_value(%FieldDescriptorProto{type: :bool, default_value: "true"}), do: true
defp get_default_value(%FieldDescriptorProto{type: :bool, default_value: "false"}), do: false
defp get_default_value(%FieldDescriptorProto{type: :string} = f), do: f.default_value
defp get_default_value(%FieldDescriptorProto{type: :bytes} = f), do: f.default_value
defp get_default_value(%FieldDescriptorProto{type: :double} = f) do
f.default_value |> Float.parse() |> elem(0)
end
defp get_default_value(%FieldDescriptorProto{type: :float} = f) do
f.default_value |> Float.parse() |> elem(0)
end
defp get_default_value(f) do
f.default_value |> Integer.parse() |> elem(0)
end
end